diff --git a/.github/workflows/ton-x86-64-linux.yml b/.github/workflows/ton-x86-64-linux.yml index fdd910001..abbe1cca4 100644 --- a/.github/workflows/ton-x86-64-linux.yml +++ b/.github/workflows/ton-x86-64-linux.yml @@ -24,7 +24,7 @@ jobs: run: | cp assembly/nix/build-linux-x86-64-nix.sh . chmod +x build-linux-x86-64-nix.sh - ./build-linux-x86-64-nix.sh + ./build-linux-x86-64-nix.sh -t - name: Simple binaries test run: | @@ -38,4 +38,4 @@ jobs: uses: actions/upload-artifact@master with: name: ton-x86_64-linux-binaries - path: artifacts \ No newline at end of file + path: artifacts diff --git a/.github/workflows/ton-x86-64-macos.yml b/.github/workflows/ton-x86-64-macos.yml index c0f907181..8c71f34a1 100644 --- a/.github/workflows/ton-x86-64-macos.yml +++ b/.github/workflows/ton-x86-64-macos.yml @@ -20,7 +20,7 @@ jobs: run: | cp assembly/nix/build-macos-nix.sh . chmod +x build-macos-nix.sh - ./build-macos-nix.sh + ./build-macos-nix.sh -t - name: Simple binaries test run: | @@ -34,4 +34,4 @@ jobs: uses: actions/upload-artifact@master with: name: ton-x86_64-macos-binaries - path: artifacts \ No newline at end of file + path: artifacts diff --git a/Changelog.md b/Changelog.md index 9fec686de..9a98e44c6 100644 --- a/Changelog.md +++ b/Changelog.md @@ -1,14 +1,20 @@ ## 2024.01 Update -1. Fixes in how gas in transactions on special accounts is accounted in block limit. Previously, gas was counted as usual, so to conduct elections that costs >30m gas block limit in masterchain was set to 37m gas. To lower the limit for safety reasons it is proposed to not count gas on special accounts. Besides `gas_max` is set to `special_gas_limit` for all types of transactions on special accounts. New behavior is activated through setting `gas_prices_v3` in `ConfigParam 20;`. +1. Fixes in how gas in transactions on special accounts is accounted in block limit. Previously, gas was counted as usual, so to conduct elections that costs >30m gas block limit in masterchain was set to 37m gas. To lower the limit for safety reasons it is proposed to caunt gas on special accounts separately. Besides `gas_max` is set to `special_gas_limit` for all types of transactions on special accounts. New behavior is activated through setting `version >= 5` in `ConfigParam 8;`. * Besides update of config temporally increases gas limit on `EQD_v9j1rlsuHHw2FIhcsCFFSD367ldfDdCKcsNmNpIRzUlu` to `special_gas_limit`, see [details](https://t.me/tonstatus/88). 2. Improvements in LS behavior * Improved detection of the state with all shards applied to decrease rate of `Block is not applied` error * Better error logs: `block not in db` and `block is not applied` separation * Fix error in proof generation for blocks after merge + * Fix most of `block is not applied` issues related to sending too recent block in Proofs + * LS now check external messages till `accept_message` (`set_gas`). 3. Improvements in DHT work and storage, CellDb, config.json ammendment, peer misbehavior detection, validator session stats collection, emulator. +4. Change in CTOS and XLOAD behavior activated through setting `version >= 5` in `ConfigParam 8;`: + * Loading "nested libraries" (i.e. a library cell that points to another library cell) throws an exception. + * Loading a library consumes gas for cell load only once (for the library cell), not twice (both for the library cell and the cell in the library). + * `XLOAD` now works differently. When it takes a library cell, it returns the cell that it points to. This allows loading "nested libraries", if needed. -Besides the work of the core team, this update is based on the efforts of @XaBbl4 (peer misbehavior detection). +Besides the work of the Core team, this update is based on the efforts of @XaBbl4 (peer misbehavior detection) and @akifoq (CTOS behavior and gas limit scheme for special accounts). ## 2023.12 Update diff --git a/assembly/cicd/jenkins/test-builds.groovy b/assembly/cicd/jenkins/test-builds.groovy index 380efedd4..960ac8db7 100644 --- a/assembly/cicd/jenkins/test-builds.groovy +++ b/assembly/cicd/jenkins/test-builds.groovy @@ -31,7 +31,7 @@ pipeline { sh ''' cp assembly/nix/build-linux-x86-64-nix.sh . chmod +x build-linux-x86-64-nix.sh - ./build-linux-x86-64-nix.sh + ./build-linux-x86-64-nix.sh -t ''' sh ''' cd artifacts @@ -69,7 +69,7 @@ pipeline { sh ''' cp assembly/nix/build-linux-arm64-nix.sh . chmod +x build-linux-arm64-nix.sh - ./build-linux-arm64-nix.sh + ./build-linux-arm64-nix.sh -t ''' sh ''' cd artifacts @@ -107,7 +107,7 @@ pipeline { sh ''' cp assembly/nix/build-macos-nix.sh . chmod +x build-macos-nix.sh - ./build-macos-nix.sh + ./build-macos-nix.sh -t ''' sh ''' cd artifacts @@ -145,7 +145,7 @@ pipeline { sh ''' cp assembly/nix/build-macos-nix.sh . chmod +x build-macos-nix.sh - ./build-macos-nix.sh + ./build-macos-nix.sh -t ''' sh ''' cd artifacts @@ -233,4 +233,4 @@ pipeline { } } } -} \ No newline at end of file +} diff --git a/assembly/nix/build-linux-arm64-nix.sh b/assembly/nix/build-linux-arm64-nix.sh index bb8591413..08817e72b 100644 --- a/assembly/nix/build-linux-arm64-nix.sh +++ b/assembly/nix/build-linux-arm64-nix.sh @@ -3,12 +3,27 @@ nix-build --version test $? -eq 0 || { echo "Nix is not installed!"; exit 1; } +with_tests=false + + +while getopts 't' flag; do + case "${flag}" in + t) with_tests=true ;; + *) break + ;; + esac +done + cp assembly/nix/linux-arm64* . cp assembly/nix/microhttpd.nix . cp assembly/nix/openssl.nix . export NIX_PATH=nixpkgs=https://github.com/nixOS/nixpkgs/archive/23.05.tar.gz -nix-build linux-arm64-static.nix +if [ "$with_tests" = true ]; then + nix-build linux-arm64-static.nix --arg testing true +else + nix-build linux-arm64-static.nix +fi mkdir artifacts cp ./result/bin/* artifacts/ chmod +x artifacts/* @@ -17,4 +32,4 @@ nix-build linux-arm64-tonlib.nix cp ./result/lib/libtonlibjson.so.0.5 artifacts/libtonlibjson.so cp ./result/lib/libemulator.so artifacts/ cp -r crypto/fift/lib artifacts/ -cp -r crypto/smartcont artifacts/ \ No newline at end of file +cp -r crypto/smartcont artifacts/ diff --git a/assembly/nix/build-linux-x86-64-nix.sh b/assembly/nix/build-linux-x86-64-nix.sh index eca6fe58b..60d31c94f 100644 --- a/assembly/nix/build-linux-x86-64-nix.sh +++ b/assembly/nix/build-linux-x86-64-nix.sh @@ -3,12 +3,28 @@ nix-build --version test $? -eq 0 || { echo "Nix is not installed!"; exit 1; } +with_tests=false + + +while getopts 't' flag; do + case "${flag}" in + t) with_tests=true ;; + *) break + ;; + esac +done + cp assembly/nix/linux-x86-64* . cp assembly/nix/microhttpd.nix . cp assembly/nix/openssl.nix . export NIX_PATH=nixpkgs=https://github.com/nixOS/nixpkgs/archive/23.05.tar.gz -nix-build linux-x86-64-static.nix +if [ "$with_tests" = true ]; then + nix-build linux-x86-64-static.nix --arg testing true +else + nix-build linux-x86-64-static.nix +fi + mkdir artifacts cp ./result/bin/* artifacts/ chmod +x artifacts/* @@ -17,4 +33,4 @@ nix-build linux-x86-64-tonlib.nix cp ./result/lib/libtonlibjson.so.0.5 artifacts/libtonlibjson.so cp ./result/lib/libemulator.so artifacts/ cp -r crypto/fift/lib artifacts/ -cp -r crypto/smartcont artifacts/ \ No newline at end of file +cp -r crypto/smartcont artifacts/ diff --git a/assembly/nix/build-macos-nix.sh b/assembly/nix/build-macos-nix.sh index fdf674a69..c3664bf02 100644 --- a/assembly/nix/build-macos-nix.sh +++ b/assembly/nix/build-macos-nix.sh @@ -3,9 +3,25 @@ nix-build --version test $? -eq 0 || { echo "Nix is not installed!"; exit 1; } +with_tests=false + + +while getopts 't' flag; do + case "${flag}" in + t) with_tests=true ;; + *) break + ;; + esac +done + cp assembly/nix/macos-* . export NIX_PATH=nixpkgs=https://github.com/nixOS/nixpkgs/archive/23.05.tar.gz -nix-build macos-static.nix + +if [ "$with_tests" = true ]; then + nix-build macos-static.nix --arg testing true +else + nix-build macos-static.nix +fi mkdir artifacts cp ./result-bin/bin/* artifacts/ chmod +x artifacts/* @@ -14,4 +30,4 @@ nix-build macos-tonlib.nix cp ./result/lib/libtonlibjson.dylib artifacts/ cp ./result/lib/libemulator.dylib artifacts/ cp -r crypto/fift/lib artifacts/ -cp -r crypto/smartcont artifacts/ \ No newline at end of file +cp -r crypto/smartcont artifacts/ diff --git a/assembly/nix/linux-arm64-static.nix b/assembly/nix/linux-arm64-static.nix index 616dfba5b..5e834269f 100644 --- a/assembly/nix/linux-arm64-static.nix +++ b/assembly/nix/linux-arm64-static.nix @@ -3,6 +3,7 @@ { pkgs ? import { system = builtins.currentSystem; } , lib ? pkgs.lib , stdenv ? pkgs.stdenv +, testing ? false }: let microhttpdmy = (import ./microhttpd.nix) {}; @@ -25,7 +26,7 @@ stdenv.mkDerivation { ]; makeStatic = true; - doCheck = true; + doCheck = testing; cmakeFlags = [ "-DTON_USE_ABSEIL=OFF" diff --git a/assembly/nix/linux-x86-64-static.nix b/assembly/nix/linux-x86-64-static.nix index 616dfba5b..5e834269f 100644 --- a/assembly/nix/linux-x86-64-static.nix +++ b/assembly/nix/linux-x86-64-static.nix @@ -3,6 +3,7 @@ { pkgs ? import { system = builtins.currentSystem; } , lib ? pkgs.lib , stdenv ? pkgs.stdenv +, testing ? false }: let microhttpdmy = (import ./microhttpd.nix) {}; @@ -25,7 +26,7 @@ stdenv.mkDerivation { ]; makeStatic = true; - doCheck = true; + doCheck = testing; cmakeFlags = [ "-DTON_USE_ABSEIL=OFF" diff --git a/assembly/nix/macos-static.nix b/assembly/nix/macos-static.nix index e65ec1a87..a3d4667f3 100644 --- a/assembly/nix/macos-static.nix +++ b/assembly/nix/macos-static.nix @@ -3,6 +3,7 @@ { pkgs ? import { system = builtins.currentSystem; } , lib ? pkgs.lib , stdenv ? pkgs.stdenv +, testing ? false }: pkgs.llvmPackages_14.stdenv.mkDerivation { @@ -29,7 +30,7 @@ pkgs.llvmPackages_14.stdenv.mkDerivation { dontAddStaticConfigureFlags = true; makeStatic = true; - doCheck = true; + doCheck = testing; configureFlags = []; @@ -62,4 +63,4 @@ pkgs.llvmPackages_14.stdenv.mkDerivation { done ''; outputs = [ "bin" "out" ]; -} \ No newline at end of file +} diff --git a/common/global-version.h b/common/global-version.h index 01c1795df..9b23d46dc 100644 --- a/common/global-version.h +++ b/common/global-version.h @@ -19,6 +19,6 @@ namespace ton { // See doc/GlobalVersions.md -const int SUPPORTED_VERSION = 4; +const int SUPPORTED_VERSION = 5; } diff --git a/crypto/block/block.tlb b/crypto/block/block.tlb index eb1f8b941..4b36f13ba 100644 --- a/crypto/block/block.tlb +++ b/crypto/block/block.tlb @@ -696,11 +696,6 @@ gas_prices_ext#de gas_price:uint64 gas_limit:uint64 special_gas_limit:uint64 gas block_gas_limit:uint64 freeze_due_limit:uint64 delete_due_limit:uint64 = GasLimitsPrices; -// same fields as gas_prices_ext; behavior differs -gas_prices_v3#df gas_price:uint64 gas_limit:uint64 special_gas_limit:uint64 gas_credit:uint64 - block_gas_limit:uint64 freeze_due_limit:uint64 delete_due_limit:uint64 - = GasLimitsPrices; - gas_flat_pfx#d1 flat_gas_limit:uint64 flat_gas_price:uint64 other:GasLimitsPrices = GasLimitsPrices; diff --git a/crypto/block/mc-config.cpp b/crypto/block/mc-config.cpp index 514021423..0cef7322f 100644 --- a/crypto/block/mc-config.cpp +++ b/crypto/block/mc-config.cpp @@ -654,13 +654,9 @@ td::Result Config::do_get_gas_limits_prices(td::Ref c res.delete_due_limit = r.delete_due_limit; }; block::gen::GasLimitsPrices::Record_gas_prices_ext rec; - block::gen::GasLimitsPrices::Record_gas_prices_v3 rec_v3; vm::CellSlice cs0 = cs; if (tlb::unpack(cs, rec)) { f(rec, rec.special_gas_limit); - } else if (tlb::unpack(cs = cs0, rec_v3)) { - f(rec_v3, rec_v3.special_gas_limit); - res.special_full_limit = true; } else { block::gen::GasLimitsPrices::Record_gas_prices rec0; if (tlb::unpack(cs = cs0, rec0)) { diff --git a/crypto/block/mc-config.h b/crypto/block/mc-config.h index dcc48b4bc..caab93f36 100644 --- a/crypto/block/mc-config.h +++ b/crypto/block/mc-config.h @@ -349,7 +349,6 @@ struct GasLimitsPrices { td::uint64 block_gas_limit{0}; td::uint64 freeze_due_limit{0}; td::uint64 delete_due_limit{0}; - bool special_full_limit{false}; td::RefInt256 compute_gas_price(td::uint64 gas_used) const; }; diff --git a/crypto/block/transaction.cpp b/crypto/block/transaction.cpp index ae50a7100..e43b93053 100644 --- a/crypto/block/transaction.cpp +++ b/crypto/block/transaction.cpp @@ -1039,12 +1039,8 @@ bool ComputePhaseConfig::parse_GasLimitsPrices_internal(Ref cs, t delete_due_limit = td::make_refint(r.delete_due_limit); }; block::gen::GasLimitsPrices::Record_gas_prices_ext rec; - block::gen::GasLimitsPrices::Record_gas_prices_v3 rec_v3; if (tlb::csr_unpack(cs, rec)) { f(rec, rec.special_gas_limit); - } else if (tlb::csr_unpack(cs, rec_v3)) { - f(rec_v3, rec_v3.special_gas_limit); - special_gas_full = true; } else { block::gen::GasLimitsPrices::Record_gas_prices rec0; if (tlb::csr_unpack(std::move(cs), rec0)) { @@ -1153,8 +1149,8 @@ namespace transaction { * not enough to clean up old queires, thus locking funds inside. * See comment in crypto/smartcont/highload-wallet-v2-code.fc for details on why this happened. * Account address: EQD_v9j1rlsuHHw2FIhcsCFFSD367ldfDdCKcsNmNpIRzUlu - * It was proposed to validators to increase gas limit for this account for a limited amount of time (until 2024-02-16). - * It is activated by setting gas_prices_v3 in ConfigParam 20 (config_mc_gas_prices). + * It was proposed to validators to increase gas limit for this account for a limited amount of time (until 2024-02-29). + * It is activated by setting global version to 5 in ConfigParam 8. * This config change also activates new behavior for special accounts in masterchain. * * @param cfg The compute phase configuration. @@ -1164,10 +1160,10 @@ namespace transaction { * @returns True if gas_limit override is required, false otherwise */ static bool override_gas_limit(const ComputePhaseConfig& cfg, ton::UnixTime now, const Account& account) { - if (!cfg.mc_gas_prices.special_full_limit) { + if (!cfg.special_gas_full) { return false; } - ton::UnixTime until = 1708041600; // 2024-02-16 00:00:00 UTC + ton::UnixTime until = 1709164800; // 2024-02-29 00:00:00 UTC ton::WorkchainId wc = 0; const char* addr_hex = "FFBFD8F5AE5B2E1C7C3614885CB02145483DFAEE575F0DD08A72C366369211CD"; return now < until && account.workchain == wc && account.addr.to_hex() == addr_hex; @@ -1563,6 +1559,7 @@ bool Transaction::prepare_compute_phase(const ComputePhaseConfig& cfg) { vm.set_global_version(cfg.global_version); vm.set_c7(prepare_vm_c7(cfg)); // tuple with SmartContractInfo vm.set_chksig_always_succeed(cfg.ignore_chksig); + vm.set_stop_on_accept_message(cfg.stop_on_accept_message); // vm.incr_stack_trace(1); // enable stack dump after each step LOG(DEBUG) << "starting VM"; @@ -3533,6 +3530,7 @@ td::Status FetchConfigParams::fetch_config_params( TRY_RESULT_PREFIX(mc_gas_prices, config.get_gas_limits_prices(true), "cannot unpack masterchain gas prices and limits: "); compute_phase_cfg->mc_gas_prices = std::move(mc_gas_prices); + compute_phase_cfg->special_gas_full = config.get_global_version() >= 5; storage_phase_cfg->enable_due_payment = config.get_global_version() >= 4; compute_phase_cfg->block_rand_seed = *rand_seed; compute_phase_cfg->max_vm_data_depth = size_limits.max_vm_data_depth; diff --git a/crypto/block/transaction.h b/crypto/block/transaction.h index 7539efe03..1ed2dfd3c 100644 --- a/crypto/block/transaction.h +++ b/crypto/block/transaction.h @@ -120,6 +120,7 @@ struct ComputePhaseConfig { std::unique_ptr suspended_addresses; SizeLimitsConfig size_limits; int vm_log_verbosity = 0; + bool stop_on_accept_message = false; ComputePhaseConfig() : gas_price(0), gas_limit(0), special_gas_limit(0), gas_credit(0) { compute_threshold(); diff --git a/crypto/vm/cellops.cpp b/crypto/vm/cellops.cpp index 9e10e072c..af84a67e5 100644 --- a/crypto/vm/cellops.cpp +++ b/crypto/vm/cellops.cpp @@ -892,6 +892,40 @@ int exec_load_special_cell(VmState* st, bool quiet) { Stack& stack = st->get_stack(); VM_LOG(st) << "execute XLOAD" << (quiet ? "Q" : ""); auto cell = stack.pop_cell(); + if (st->get_global_version() >= 5) { + st->register_cell_load(cell->get_hash()); + auto r_loaded_cell = cell->load_cell(); + if (r_loaded_cell.is_error()) { + if (quiet) { + stack.push_bool(false); + return 0; + } else { + throw VmError{Excno::cell_und, "failed to load cell"}; + } + } + auto loaded_cell = r_loaded_cell.move_as_ok(); + if (loaded_cell.data_cell->is_special()) { + if (loaded_cell.data_cell->special_type() != CellTraits::SpecialType::Library) { + if (quiet) { + stack.push_bool(false); + return 0; + } else { + throw VmError{Excno::cell_und, "unexpected special cell"}; + } + } + CellSlice cs(std::move(loaded_cell)); + DCHECK(cs.size() == Cell::hash_bits + 8); + cell = st->load_library(cs.data_bits() + 8); + if (cell.is_null()) { + if (quiet) { + stack.push_bool(false); + return 0; + } else { + throw VmError{Excno::cell_und, "failed to load library cell"}; + } + } + } + } stack.push_cell(cell); if (quiet) { stack.push_bool(true); diff --git a/crypto/vm/cells/CellSlice.cpp b/crypto/vm/cells/CellSlice.cpp index e1df57598..ee5f6941a 100644 --- a/crypto/vm/cells/CellSlice.cpp +++ b/crypto/vm/cells/CellSlice.cpp @@ -1056,9 +1056,10 @@ std::ostream& operator<<(std::ostream& os, Ref cs_ref) { // If can_be_special is not null, then it is allowed to load special cell // Flag whether loaded cell is actually special will be stored into can_be_special VirtualCell::LoadedCell load_cell_slice_impl(Ref cell, bool* can_be_special) { + auto* vm_state_interface = VmStateInterface::get(); + bool library_loaded = false; while (true) { - auto* vm_state_interface = VmStateInterface::get(); - if (vm_state_interface) { + if (vm_state_interface && !library_loaded) { vm_state_interface->register_cell_load(cell->get_hash()); } auto r_loaded_cell = cell->load_cell(); @@ -1077,6 +1078,12 @@ VirtualCell::LoadedCell load_cell_slice_impl(Ref cell, bool* can_be_specia } else if (loaded_cell.data_cell->is_special()) { if (loaded_cell.data_cell->special_type() == DataCell::SpecialType::Library) { if (vm_state_interface) { + if (vm_state_interface->get_global_version() >= 5) { + if (library_loaded) { + throw VmError{Excno::cell_und, "failed to load library cell: recursive library cells are not allowed"}; + } + library_loaded = true; + } CellSlice cs(std::move(loaded_cell)); DCHECK(cs.size() == Cell::hash_bits + 8); auto library_cell = vm_state_interface->load_library(cs.data_bits() + 8); diff --git a/crypto/vm/tonops.cpp b/crypto/vm/tonops.cpp index d150f30b3..dce617971 100644 --- a/crypto/vm/tonops.cpp +++ b/crypto/vm/tonops.cpp @@ -67,6 +67,10 @@ int exec_set_gas_generic(VmState* st, long long new_gas_limit) { throw VmNoGas{}; } st->change_gas_limit(new_gas_limit); + if (st->get_stop_on_accept_message()) { + VM_LOG(st) << "External message is accepted, stopping TVM"; + return st->jump(td::Ref{true, 0}); + } return 0; } diff --git a/crypto/vm/vm.h b/crypto/vm/vm.h index 2066db4c1..e5cca026c 100644 --- a/crypto/vm/vm.h +++ b/crypto/vm/vm.h @@ -98,6 +98,7 @@ class VmState final : public VmStateInterface { td::HashSet loaded_cells; int stack_trace{0}, debug_off{0}; bool chksig_always_succeed{false}; + bool stop_on_accept_message{false}; td::optional missing_library; td::uint16 max_data_depth = 512; // Default value int global_version{0}; @@ -339,7 +340,7 @@ class VmState final : public VmStateInterface { void preclear_cr(const ControlRegs& save) { cr &= save; } - int get_global_version() const { + int get_global_version() const override { return global_version; } void set_global_version(int version) { @@ -381,6 +382,12 @@ class VmState final : public VmStateInterface { bool get_chksig_always_succeed() const { return chksig_always_succeed; } + void set_stop_on_accept_message(bool flag) { + stop_on_accept_message = flag; + } + bool get_stop_on_accept_message() const { + return stop_on_accept_message; + } Ref ref_to_cont(Ref cell) const { return td::make_ref(load_cell_slice_ref(std::move(cell)), get_cp()); } diff --git a/crypto/vm/vmstate.h b/crypto/vm/vmstate.h index a81a4e78d..0d6c3fdf9 100644 --- a/crypto/vm/vmstate.h +++ b/crypto/vm/vmstate.h @@ -19,6 +19,7 @@ #pragma once #include "common/refcnt.hpp" #include "vm/cells.h" +#include "common/global-version.h" #include "td/utils/Context.h" @@ -38,6 +39,9 @@ class VmStateInterface : public td::Context { virtual bool register_op(int op_units = 1) { return true; }; + virtual int get_global_version() const { + return ton::SUPPORTED_VERSION; + } }; } // namespace vm diff --git a/doc/GlobalVersions.md b/doc/GlobalVersions.md index ccfca9406..d2064b901 100644 --- a/doc/GlobalVersions.md +++ b/doc/GlobalVersions.md @@ -36,4 +36,21 @@ intermediate value before division (e.g. `(xy+w)/z`). * Flag +16 in actions "Send message", "Reserve", "Change library" causes bounce if action fails. ### Storage phase -* Unpaid storage fee is now saved to `due_payment` \ No newline at end of file +* Unpaid storage fee is now saved to `due_payment` + +## Version 5 + +### Gas limits +Version 5 enables higher gas limits for special contracts. + +* Gas limit for all transactions on special contracts is set to `special_gas_limit` from `ConfigParam 20` (which is 35M at the moment of writing). +Previously only ticktock transactions had this limit, while ordinary transactions had a default limit of `gas_limit` gas (1M). +* Gas usage of special contracts is not taken into account when checking block limits. This allows keeping masterchain block limits low +while having high gas limits for elector. +* Gas limit on `EQD_v9j1rlsuHHw2FIhcsCFFSD367ldfDdCKcsNmNpIRzUlu` is increased to `special_gas_limit * 2` until 2024-02-29. +See [this post](https://t.me/tonstatus/88) for details. + +### Loading libraries +* Loading "nested libraries" (i.e. a library cell that points to another library cell) throws an exception. +* Loading a library consumes gas for cell load only once (for the library cell), not twice (both for the library cell and the cell in the library). +* `XLOAD` now works differently. When it takes a library cell, it returns the cell that it points to. This allows loading "nested libraries", if needed. \ No newline at end of file diff --git a/recent_changelog.md b/recent_changelog.md index 637f416c6..852fb7644 100644 --- a/recent_changelog.md +++ b/recent_changelog.md @@ -1,11 +1,17 @@ ## 2024.01 Update -1. Fixes in how gas in transactions on special accounts is accounted in block limit. Previously, gas was counted as usual, so to conduct elections that costs >30m gas block limit in masterchain was set to 37m gas. To lower the limit for safety reasons it is proposed to not count gas on special accounts. Besides `gas_max` is set to `special_gas_limit` for all types of transactions on special accounts. New behavior is activated through setting `gas_prices_v3` in `ConfigParam 20;`. +1. Fixes in how gas in transactions on special accounts is accounted in block limit. Previously, gas was counted as usual, so to conduct elections that costs >30m gas block limit in masterchain was set to 37m gas. To lower the limit for safety reasons it is proposed to caunt gas on special accounts separately. Besides `gas_max` is set to `special_gas_limit` for all types of transactions on special accounts. New behavior is activated through setting `version >= 5` in `ConfigParam 8;`. * Besides update of config temporally increases gas limit on `EQD_v9j1rlsuHHw2FIhcsCFFSD367ldfDdCKcsNmNpIRzUlu` to `special_gas_limit`, see [details](https://t.me/tonstatus/88). 2. Improvements in LS behavior * Improved detection of the state with all shards applied to decrease rate of `Block is not applied` error * Better error logs: `block not in db` and `block is not applied` separation * Fix error in proof generation for blocks after merge + * Fix most of `block is not applied` issues related to sending too recent block in Proofs + * LS now check external messages till `accept_message` (`set_gas`). 3. Improvements in DHT work and storage, CellDb, config.json ammendment, peer misbehavior detection, validator session stats collection, emulator. +4. Change in CTOS and XLOAD behavior activated through setting `version >= 5` in `ConfigParam 8;`: + * Loading "nested libraries" (i.e. a library cell that points to another library cell) throws an exception. + * Loading a library consumes gas for cell load only once (for the library cell), not twice (both for the library cell and the cell in the library). + * `XLOAD` now works differently. When it takes a library cell, it returns the cell that it points to. This allows loading "nested libraries", if needed. -Besides the work of the core team, this update is based on the efforts of @XaBbl4 (peer misbehavior detection). +Besides the work of the Core team, this update is based on the efforts of @XaBbl4 (peer misbehavior detection) and @akifoq (CTOS behavior and gas limit scheme for special accounts). diff --git a/test/test-validator-session-state.cpp b/test/test-validator-session-state.cpp index 5ed08add7..cb5d817f2 100644 --- a/test/test-validator-session-state.cpp +++ b/test/test-validator-session-state.cpp @@ -34,6 +34,7 @@ #include "validator-session/validator-session-description.h" #include "validator-session/validator-session-state.h" +#include "validator-session/validator-session-description.hpp" #include #include @@ -48,17 +49,13 @@ class Description : public ton::validatorsession::ValidatorSessionDescription { return 0; } void *alloc(size_t size, size_t align, bool temp) override { - size = (size + 15) / 16 * 16; - td::uint32 idx = temp ? 1 : 0; - auto s = pdata_cur_[idx].fetch_add(size); - CHECK(s + size <= pdata_size_[idx]); - return static_cast(pdata_[idx] + s); + return (temp ? mem_temp_ : mem_perm_).alloc(size, align); } bool is_persistent(const void *ptr) const override { - return ptr == nullptr || (ptr >= pdata_[0] && ptr < pdata_[0] + pdata_size_[0]); + return mem_perm_.contains(ptr); } void clear_temp_memory() override { - pdata_cur_[1] = 0; + mem_temp_.clear(); } ton::PublicKeyHash get_source_id(td::uint32 idx) const override { @@ -189,21 +186,8 @@ class Description : public ton::validatorsession::ValidatorSessionDescription { return opts_; } - ~Description() { - delete[] pdata_[0]; - delete[] pdata_[1]; - } - Description(ton::validatorsession::ValidatorSessionOptions opts, td::uint32 total_nodes) - : opts_(opts), total_nodes_(total_nodes) { - pdata_size_[0] = - static_cast(std::numeric_limits::max() < (1ull << 32) ? 1ull << 30 : 1ull << 33); - pdata_size_[1] = 1 << 22; - pdata_[0] = new td::uint8[pdata_size_[0]]; - pdata_[1] = new td::uint8[pdata_size_[1]]; - pdata_cur_[0] = 0; - pdata_cur_[1] = 0; - + : opts_(opts), total_nodes_(total_nodes), mem_perm_(1 << 30), mem_temp_(1 << 22) { for (auto &el : cache_) { Cached v{nullptr}; el.store(v, std::memory_order_relaxed); @@ -224,9 +208,7 @@ class Description : public ton::validatorsession::ValidatorSessionDescription { }; std::array, cache_size> cache_; - td::uint8 *pdata_[2]; - std::atomic pdata_cur_[2]; - size_t pdata_size_[2]; + ton::validatorsession::ValidatorSessionDescriptionImpl::MemPool mem_perm_, mem_temp_; }; double myrand() { diff --git a/validator-session/validator-session-description.hpp b/validator-session/validator-session-description.hpp index 0c662b7a2..5e09c694a 100644 --- a/validator-session/validator-session-description.hpp +++ b/validator-session/validator-session-description.hpp @@ -58,6 +58,7 @@ class ValidatorSessionDescriptionImpl : public ValidatorSessionDescription { }; std::array, cache_size> cache_; + public: class MemPool { public: explicit MemPool(size_t chunk_size); @@ -71,6 +72,8 @@ class ValidatorSessionDescriptionImpl : public ValidatorSessionDescription { std::vector data_; size_t ptr_ = 0; }; + + private: MemPool mem_perm_ = MemPool(mem_chunk_size_perm); MemPool mem_temp_ = MemPool(mem_chunk_size_temp); diff --git a/validator/impl/collator-impl.h b/validator/impl/collator-impl.h index fe931cd66..c6ba27b45 100644 --- a/validator/impl/collator-impl.h +++ b/validator/impl/collator-impl.h @@ -244,7 +244,7 @@ class Collator final : public td::actor::Actor { Ref& in_msg); bool create_ticktock_transactions(int mask); bool create_ticktock_transaction(const ton::StdSmcAddress& smc_addr, ton::LogicalTime req_start_lt, int mask); - Ref create_ordinary_transaction(Ref msg_root); + Ref create_ordinary_transaction(Ref msg_root, bool is_special_tx = false); bool check_cur_validator_set(); bool unpack_last_mc_state(); bool unpack_last_state(); diff --git a/validator/impl/collator.cpp b/validator/impl/collator.cpp index 1b7991f6e..413c5be98 100644 --- a/validator/impl/collator.cpp +++ b/validator/impl/collator.cpp @@ -2667,8 +2667,7 @@ bool Collator::create_ticktock_transaction(const ton::StdSmcAddress& smc_addr, t return fatal_error(td::Status::Error( -666, std::string{"cannot serialize new transaction for smart contract "} + smc_addr.to_hex())); } - if (!trans->update_limits(*block_limit_status_, - /* with_gas = */ !(acc->is_special && compute_phase_cfg_.special_gas_full))) { + if (!trans->update_limits(*block_limit_status_, /* with_gas = */ false)) { return fatal_error(-666, "cannot update block limit status to include the new transaction"); } if (trans->commit(*acc).is_null()) { @@ -2684,10 +2683,11 @@ bool Collator::create_ticktock_transaction(const ton::StdSmcAddress& smc_addr, t * Creates an ordinary transaction using a given message. * * @param msg_root The root of the message to be processed serialized using Message TLB-scheme. + * @param is_special_tx True if creating a special transaction (mint/recover), false otherwise. * * @returns The root of the serialized transaction, or an empty reference if the transaction creation fails. */ -Ref Collator::create_ordinary_transaction(Ref msg_root) { +Ref Collator::create_ordinary_transaction(Ref msg_root, bool is_special_tx) { ton::StdSmcAddress addr; auto cs = vm::load_cell_slice(msg_root); bool external; @@ -2746,7 +2746,7 @@ Ref Collator::create_ordinary_transaction(Ref msg_root) { std::unique_ptr trans = res.move_as_ok(); if (!trans->update_limits(*block_limit_status_, - /* with_gas = */ !(acc->is_special && compute_phase_cfg_.special_gas_full))) { + /* with_gas = */ !(is_special_tx && compute_phase_cfg_.special_gas_full))) { fatal_error("cannot update block limit status to include the new transaction"); return {}; } @@ -3019,7 +3019,7 @@ int Collator::process_one_new_message(block::NewOutMsg msg, bool enqueue_only, R return -1; } // 1. create a Transaction processing this Message - auto trans_root = create_ordinary_transaction(msg.msg); + auto trans_root = create_ordinary_transaction(msg.msg, is_special != nullptr); if (trans_root.is_null()) { fatal_error("cannot create transaction for re-processing output message"); return -1; diff --git a/validator/impl/external-message.cpp b/validator/impl/external-message.cpp index e7665afc2..073e7360e 100644 --- a/validator/impl/external-message.cpp +++ b/validator/impl/external-message.cpp @@ -156,6 +156,7 @@ td::Status ExtMessageQ::run_message_on_account(ton::WorkchainId wc, } compute_phase_cfg_.libraries = std::make_unique(config->get_libraries_root(), 256); compute_phase_cfg_.with_vm_log = true; + compute_phase_cfg_.stop_on_accept_message = true; auto res = Collator::impl_create_ordinary_transaction(msg_root, acc, utime, lt, &storage_phase_cfg_, &compute_phase_cfg_, diff --git a/validator/impl/validate-query.cpp b/validator/impl/validate-query.cpp index 94eded6cd..2b3ccd85a 100644 --- a/validator/impl/validate-query.cpp +++ b/validator/impl/validate-query.cpp @@ -945,6 +945,7 @@ bool ValidateQuery::fetch_config_params() { return fatal_error(mc_gas_prices.move_as_error_prefix("cannot unpack masterchain gas prices and limits: ")); } compute_phase_cfg_.mc_gas_prices = mc_gas_prices.move_as_ok(); + compute_phase_cfg_.special_gas_full = config_->get_global_version() >= 5; storage_phase_cfg_.enable_due_payment = config_->get_global_version() >= 4; compute_phase_cfg_.block_rand_seed = rand_seed_; compute_phase_cfg_.libraries = std::make_unique(config_->get_libraries_root(), 256); @@ -4684,6 +4685,7 @@ bool ValidateQuery::check_one_transaction(block::Account& account, ton::LogicalT bool external{false}, ihr_delivered{false}, need_credit_phase{false}; // check input message block::CurrencyCollection money_imported(0), money_exported(0); + bool is_special_tx = false; // recover/mint transaction if (in_msg_root.not_null()) { auto in_descr_cs = in_msg_dict_->lookup(in_msg_root->get_hash().as_bitslice()); if (in_descr_cs.is_null()) { @@ -4699,6 +4701,7 @@ bool ValidateQuery::check_one_transaction(block::Account& account, ton::LogicalT << " has an invalid InMsg record (not one of msg_import_ext, msg_import_fin, " "msg_import_imm or msg_import_ihr)"); } + is_special_tx = is_special_in_msg(*in_descr_cs); // once we know there is a InMsg with correct hash, we already know that it contains a message with this hash (by the verification of InMsg), so it is our message // have still to check its destination address and imported value // and that it refers to this transaction @@ -4716,7 +4719,7 @@ bool ValidateQuery::check_one_transaction(block::Account& account, ton::LogicalT << " processed inbound message created later at logical time " << info.created_lt); } - if (info.created_lt != start_lt_ || !is_special_in_msg(*in_descr_cs)) { + if (info.created_lt != start_lt_ || !is_special_tx) { msg_proc_lt_.emplace_back(addr, lt, info.created_lt); } dest = std::move(info.dest); @@ -5056,19 +5059,31 @@ bool ValidateQuery::check_one_transaction(block::Account& account, ton::LogicalT return reject_query(PSTRING() << "cannot re-create the serialization of transaction " << lt << " for smart contract " << addr.to_hex()); } - if (!trs->update_limits(*block_limit_status_, - /* with_gas = */ !account.is_special && !trs->gas_limit_overridden, - /* with_size = */ false)) { + if (!trs->update_limits(*block_limit_status_, /* with_gas = */ false, /* with_size = */ false)) { return fatal_error(PSTRING() << "cannot update block limit status to include transaction " << lt << " of account " << addr.to_hex()); } - if (block_limit_status_->gas_used > block_limits_->gas.hard() + compute_phase_cfg_.gas_limit) { - // Note that block_limit_status_->gas_used does not include transactions in special accounts + + // Collator should stop if total gas usage exceeds limits, including transactions on special accounts, but without + // ticktocks and mint/recover. + // Here Validator checks a weaker condition + if (!is_special_tx && !trs->gas_limit_overridden && trans_type == block::transaction::Transaction::tr_ord) { + (account.is_special ? total_special_gas_used_ : total_gas_used_) += trs->gas_used(); + } + if (total_gas_used_ > block_limits_->gas.hard() + compute_phase_cfg_.gas_limit) { return reject_query(PSTRING() << "gas block limits are exceeded: total_gas_used > gas_limit_hard + trx_gas_limit (" - << "total_gas_used=" << block_limit_status_->gas_used + << "total_gas_used=" << total_gas_used_ << ", gas_limit_hard=" << block_limits_->gas.hard() << ", trx_gas_limit=" << compute_phase_cfg_.gas_limit << ")"); } + if (total_special_gas_used_ > block_limits_->gas.hard() + compute_phase_cfg_.special_gas_limit) { + return reject_query( + PSTRING() << "gas block limits are exceeded: total_special_gas_used > gas_limit_hard + special_gas_limit (" + << "total_special_gas_used=" << total_special_gas_used_ + << ", gas_limit_hard=" << block_limits_->gas.hard() + << ", special_gas_limit=" << compute_phase_cfg_.special_gas_limit << ")"); + } + auto trans_root2 = trs->commit(account); if (trans_root2.is_null()) { return reject_query(PSTRING() << "the re-created transaction " << lt << " for smart contract " << addr.to_hex() diff --git a/validator/impl/validate-query.hpp b/validator/impl/validate-query.hpp index ff8cc83c8..8829ac61f 100644 --- a/validator/impl/validate-query.hpp +++ b/validator/impl/validate-query.hpp @@ -195,6 +195,7 @@ class ValidateQuery : public td::actor::Actor { ton::LogicalTime prev_key_block_lt_; std::unique_ptr block_limits_; std::unique_ptr block_limit_status_; + td::uint64 total_gas_used_{0}, total_special_gas_used_{0}; LogicalTime start_lt_, end_lt_; UnixTime prev_now_{~0u}, now_{~0u}; diff --git a/validator/manager.cpp b/validator/manager.cpp index 21fa5887c..af01ff8ae 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -2460,8 +2460,8 @@ void ValidatorManagerImpl::update_shard_client_state(BlockIdExt masterchain_bloc } void ValidatorManagerImpl::get_shard_client_state(bool from_db, td::Promise promise) { - if (!shard_client_.empty() && !from_db) { - td::actor::send_closure(shard_client_, &ShardClient::get_processed_masterchain_block_id, std::move(promise)); + if (shard_client_handle_ && !from_db) { + promise.set_result(shard_client_handle_->id()); } else { td::actor::send_closure(db_, &Db::get_shard_client_state, std::move(promise)); }