Merge pull request #371 from ton-blockchain/release-candidate

Merge updates

Author: EmelyanenkoK

CMakeLists.txt

@@ -423,6 +423,9 @@ target_link_libraries(test-vm PRIVATE ton_crypto fift-lib)
 add_executable(test-smartcont test/test-td-main.cpp ${SMARTCONT_TEST_SOURCE})
 target_link_libraries(test-smartcont PRIVATE smc-envelope fift-lib ton_db)
 
+add_executable(test-bigint ${BIGINT_TEST_SOURCE})
+target_link_libraries(test-bigint PRIVATE ton_crypto)
+
 add_executable(test-cells test/test-td-main.cpp ${CELLS_TEST_SOURCE})
 target_link_libraries(test-cells PRIVATE ton_crypto)
 
@@ -523,6 +526,7 @@ if (HAS_PARENT)
           ${FEC_TEST_SOURCE}
           ${ED25519_TEST_SOURCE}
           ${TONDB_TEST_SOURCE}
+          ${BIGNUM_TEST_SOURCE}
           ${CELLS_TEST_SOURCE} # ${TONVM_TEST_SOURCE} ${FIFT_TEST_SOURCE} ${TONLIB_ONLINE_TEST_SOURCE}
           PARENT_SCOPE)
 endif()
@@ -536,6 +540,7 @@ set(TEST_OPTIONS "--regression ${CMAKE_CURRENT_SOURCE_DIR}/test/regression-tests
 separate_arguments(TEST_OPTIONS)
 add_test(test-ed25519-crypto crypto/test-ed25519-crypto)
 add_test(test-ed25519 test-ed25519)
+add_test(test-bigint test-bigint)
 add_test(test-vm test-vm ${TEST_OPTIONS})
 add_test(test-fift test-fift ${TEST_OPTIONS})
 add_test(test-cells test-cells ${TEST_OPTIONS})

Changelog.md

@@ -0,0 +1,14 @@
+## 05.2022 Update
+* Initial synchronization improved: adjusted timeouts for state download and the way of choosing which state to download. Nodes with low network speed and/or bad connectivity will synchronize faster and consistently.
+* Improved peer-to-peer network stability and DDoS resistance: now peers will only relay valid messages to the network. Large messages, which require splitting for relaying, will be retranslated as well, but only after the node gets all parts, and reassembles and checks them. Validators may sign certificates for network peers, which allow relaying large messages by parts without checks. It is used now by validators to faster relay new blocks. Sign and import certificate commands are exposed via `validator-engine-console`.
+* Fixed some rare edge cases in TVM arithmetic operations related to big numbers (`2**63+`)
+* Improved fixes used to combat wrong activate-destruct-activate contract behavior last November.
+* Improved tonlib: support libraries (with client-side caching), getmethods completely fill c7 register, getmethods support slice arguments, improved messages listing for transactions, added extended block header params, added getConfig method.
+* RocksDB updated to a newer version.
+* Improved persistent state serialization: memory usage during serialization was optimized; the start of serialization on different nodes was sparsed.
+* FunC update: support for string literals and constants (including precompiled constant expressions), semver, `include` expressions.
+* Fixed rarely manifested bugs in `Asm.fif`.
+* LiteClient supports key as cli parameter.
+* Improved Liteserver DoS resistance for running getmethods.
+
+Besides the work of the core team, this update is based on the efforts of @tvorogme (added support for slice arguments and noted bugs in Asm.fif), @akifoq (fixed bug in Asm.fif), @cryshado (noted strange behavior of LS, which, upon inspection, turned out to be a vector of DoS attack).

catchain/catchain-receiver.cpp

@@ -486,7 +486,7 @@ void CatChainReceiverImpl::start_up() {
   }
   td::actor::send_closure(overlay_manager_, &overlay::Overlays::create_private_overlay,
                           get_source(local_idx_)->get_adnl_id(), overlay_full_id_.clone(), std::move(ids),
-                          make_callback(), overlay::OverlayPrivacyRules{0, std::move(root_keys)});
+                          make_callback(), overlay::OverlayPrivacyRules{0, 0, std::move(root_keys)});
 
   CHECK(root_block_);
 

crypto/CMakeLists.txt

@@ -259,6 +259,11 @@ set(FIFT_TEST_SOURCE
   PARENT_SCOPE
 )
 
+set(BIGINT_TEST_SOURCE
+  ${CMAKE_CURRENT_SOURCE_DIR}/test/test-bigint.cpp
+  PARENT_SCOPE
+)
+
 
 add_library(ton_crypto STATIC ${TON_CRYPTO_SOURCE})
 target_include_directories(ton_crypto PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>
@@ -293,9 +298,14 @@ add_library(src_parser ${PARSER_SOURCE})
 target_include_directories(src_parser PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>)
 target_link_libraries(src_parser PUBLIC ton_crypto)
 
+add_library(ton_block ${BLOCK_SOURCE})
+target_include_directories(ton_block PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>
+        $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/block> $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/..>)
+target_link_libraries(ton_block PUBLIC ton_crypto tdutils tdactor tl_api)
+
 add_executable(func func/func.cpp ${FUNC_LIB_SOURCE})
 target_include_directories(func PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>)
-target_link_libraries(func PUBLIC ton_crypto src_parser git)
+target_link_libraries(func PUBLIC ton_crypto src_parser git ton_block)
 if (WINGETOPT_FOUND)
   target_link_libraries_system(func wingetopt)
 endif()
@@ -319,11 +329,6 @@ if (WINGETOPT_FOUND)
   target_link_libraries_system(pow-miner wingetopt)
 endif()
 
-add_library(ton_block ${BLOCK_SOURCE})
-target_include_directories(ton_block PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>
-  $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/block> $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/..>)
-target_link_libraries(ton_block PUBLIC ton_crypto tdutils tdactor tl_api)
-
 set(TURN_OFF_LSAN cd .)
 if (TON_USE_ASAN AND NOT WIN32)
   set(TURN_OFF_LSAN export LSAN_OPTIONS=detect_leaks=0)

crypto/block/mc-config.cpp

@@ -2100,7 +2100,7 @@ Ref<vm::Cell> ConfigInfo::lookup_library(td::ConstBitPtr root_hash) const {
     return {};
   }
   auto csr = libraries_dict_->lookup(root_hash, 256);
-  if (csr.is_null() || csr->prefetch_ulong(8) != 0 || !csr->have_refs()) {  // shared_lib_descr$00 lib:^Cell
+  if (csr.is_null() || csr->prefetch_ulong(2) != 0 || !csr->have_refs()) {  // shared_lib_descr$00 lib:^Cell
     return {};
   }
   auto lib = csr->prefetch_ref();

crypto/block/transaction.cpp

@@ -271,7 +271,7 @@ bool Account::recompute_tmp_addr(Ref<vm::CellSlice>& tmp_addr, int split_depth,
 }
 
 bool Account::init_rewrite_addr(int split_depth, td::ConstBitPtr orig_addr_rewrite) {
-  if (split_depth_set_ || !created || !set_split_depth(split_depth)) {
+  if (split_depth_set_ || !set_split_depth(split_depth)) {
     return false;
   }
   addr_orig = addr;
@@ -304,15 +304,8 @@ bool Account::unpack(Ref<vm::CellSlice> shard_account, Ref<vm::CellSlice> extra,
   total_state = orig_total_state = account;
   auto acc_cs = load_cell_slice(std::move(account));
   if (block::gen::t_Account.get_tag(acc_cs) == block::gen::Account::account_none) {
-    status = acc_nonexist;
-    last_paid = 0;
-    last_trans_end_lt_ = 0;
     is_special = special;
-    if (workchain != ton::workchainInvalid) {
-      addr_orig = addr;
-      addr_rewrite = addr.cbits();
-    }
-    return compute_my_addr() && acc_cs.size_ext() == 1;
+    return acc_cs.size_ext() == 1 && init_new(now);
   }
   block::gen::Account::Record_account acc;
   block::gen::AccountStorage::Record storage;
@@ -328,6 +321,7 @@ bool Account::unpack(Ref<vm::CellSlice> shard_account, Ref<vm::CellSlice> extra,
     case block::gen::AccountState::account_uninit:
       status = orig_status = acc_uninit;
       state_hash = addr;
+      forget_split_depth();
       break;
     case block::gen::AccountState::account_frozen:
       status = orig_status = acc_frozen;
@@ -396,10 +390,42 @@ bool Account::init_new(ton::UnixTime now) {
   state_hash = addr_orig;
   status = orig_status = acc_nonexist;
   split_depth_set_ = false;
-  created = true;
   return true;
 }
 
+bool Account::forget_split_depth() {
+  split_depth_set_ = false;
+  split_depth_ = 0;
+  addr_orig = addr;
+  my_addr = my_addr_exact;
+  addr_rewrite = addr.bits();
+  return true;
+}
+
+bool Account::deactivate() {
+  if (status == acc_active) {
+    return false;
+  }
+  // forget special (tick/tock) info
+  tick = tock = false;
+  if (status == acc_nonexist || status == acc_uninit) {
+    // forget split depth and address rewriting info
+    forget_split_depth();
+    // forget specific state hash for deleted or uninitialized accounts (revert to addr)
+    state_hash = addr;
+  }
+  // forget code and data (only active accounts remember these)
+  code.clear();
+  data.clear();
+  library.clear();
+  // if deleted, balance must be zero
+  if (status == acc_nonexist && !balance.is_zero()) {
+    return false;
+  }
+  return true;
+}
+
+
 bool Account::belongs_to_shard(ton::ShardIdFull shard) const {
   return workchain == shard.workchain && ton::shard_is_ancestor(shard.shard, addr);
 }
@@ -2214,7 +2240,7 @@ Ref<vm::Cell> Transaction::commit(Account& acc) {
   CHECK((const void*)&acc == (const void*)&account);
   // export all fields modified by the Transaction into original account
   // NB: this is the only method that modifies account
-  if (orig_addr_rewrite_set && new_split_depth >= 0 && acc.status == Account::acc_nonexist &&
+  if (orig_addr_rewrite_set && new_split_depth >= 0 && acc.status != Account::acc_active &&
       acc_status == Account::acc_active) {
     LOG(DEBUG) << "setting address rewriting info for newly-activated account " << acc.addr.to_hex()
                << " with split_depth=" << new_split_depth
@@ -2243,9 +2269,7 @@ Ref<vm::Cell> Transaction::commit(Account& acc) {
     acc.tick = new_tick;
     acc.tock = new_tock;
   } else {
-    acc.tick = acc.tock = false;
-    acc.split_depth_set_ = false;
-    acc.created = true;
+    CHECK(acc.deactivate());
   }
   end_lt = 0;
   acc.push_transaction(root, start_lt);

crypto/block/transaction.h

@@ -213,17 +213,16 @@ struct Account {
   bool is_special{false};
   bool tick{false};
   bool tock{false};
-  bool created{false};
   bool split_depth_set_{false};
   unsigned char split_depth_{0};
   int verbosity{3 * 0};
   ton::UnixTime now_{0};
   ton::WorkchainId workchain{ton::workchainInvalid};
   td::BitArray<32> addr_rewrite;     // rewrite (anycast) data, split_depth bits
-  ton::StdSmcAddress addr;           // rewritten address (by replacing a prefix of `addr_orig` with `addr_rewrite`)
-  ton::StdSmcAddress addr_orig;      // address indicated in smart-contract data
-  Ref<vm::CellSlice> my_addr;        // address as stored in the smart contract (MsgAddressInt)
-  Ref<vm::CellSlice> my_addr_exact;  // exact address without anycast info
+  ton::StdSmcAddress addr;           // rewritten address (by replacing a prefix of `addr_orig` with `addr_rewrite`); it is the key in ShardAccounts
+  ton::StdSmcAddress addr_orig;      // address indicated in smart-contract data (must coincide with hash of StateInit)
+  Ref<vm::CellSlice> my_addr;        // address as stored in the smart contract (MsgAddressInt); corresponds to `addr_orig` + anycast info
+  Ref<vm::CellSlice> my_addr_exact;  // exact address without anycast info; corresponds to `addr` and has no anycast (rewrite) info
   ton::LogicalTime last_trans_end_lt_;
   ton::LogicalTime last_trans_lt_;
   ton::Bits256 last_trans_hash_;
@@ -250,6 +249,7 @@ struct Account {
   bool set_address(ton::WorkchainId wc, td::ConstBitPtr new_addr);
   bool unpack(Ref<vm::CellSlice> account, Ref<vm::CellSlice> extra, ton::UnixTime now, bool special = false);
   bool init_new(ton::UnixTime now);
+  bool deactivate();
   bool recompute_tmp_addr(Ref<vm::CellSlice>& tmp_addr, int split_depth, td::ConstBitPtr orig_addr_rewrite) const;
   td::RefInt256 compute_storage_fees(ton::UnixTime now, const std::vector<block::StoragePrices>& pricing) const;
   bool is_masterchain() const {
@@ -268,6 +268,7 @@ struct Account {
   friend struct Transaction;
   bool set_split_depth(int split_depth);
   bool check_split_depth(int split_depth) const;
+  bool forget_split_depth();
   bool init_rewrite_addr(int split_depth, td::ConstBitPtr orig_addr_rewrite);
 
  private:

crypto/common/bigint.hpp

@@ -264,7 +264,7 @@ class AnyIntView {
     return digits[size() - 1];
   }
   double top_double() const {
-    return size() > 1 ? (double)digits[size() - 1] + (double)digits[size() - 2] * (1.0 / Tr::Base)
+    return size() > 1 ? (double)digits[size() - 1] + (double)digits[size() - 2] * Tr::InvBase
                       : (double)digits[size() - 1];
   }
   bool is_odd_any() const {
@@ -314,8 +314,15 @@ class BigIntG {
     digits[0] = x;
   }
   BigIntG(Normalize, word_t x) : n(1) {
-    digits[0] = x;
-    normalize_bool();
+    if (x >= -Tr::Half && x < Tr::Half) {
+      digits[0] = x;
+    } else if (len <= 1) {
+      digits[0] = x;
+      normalize_bool();
+    } else {
+      digits[0] = ((x + Tr::Half) & (Tr::Base - 1)) - Tr::Half;
+      digits[n++] = (x >> Tr::word_shift) + (digits[0] < 0);
+    }
   }
   BigIntG(const BigIntG& x) : n(x.n) {
     std::memcpy(digits, x.digits, n * sizeof(word_t));
@@ -757,7 +764,7 @@ bool AnyIntView<Tr>::add_pow2_any(int exponent, int factor) {
   while (size() <= k) {
     digits[inc_size()] = 0;
   }
-  digits[k] += (factor << dm.rem);
+  digits[k] += ((word_t)factor << dm.rem);
   return true;
 }
 
@@ -1087,12 +1094,16 @@ int AnyIntView<Tr>::cmp_any(const AnyIntView<Tr>& yp) const {
 
 template <class Tr>
 int AnyIntView<Tr>::cmp_any(word_t y) const {
-  if (size() > 1) {
-    return top_word() < 0 ? -1 : 1;
-  } else if (size() == 1) {
+  if (size() == 1) {
     return digits[0] < y ? -1 : (digits[0] > y ? 1 : 0);
-  } else {
+  } else if (!size()) {
     return 0x80000000;
+  } else if (size() == 2 && (y >= Tr::Half || y < -Tr::Half)) {
+    word_t x0 = digits[0] & (Tr::Base - 1), y0 = y & (Tr::Base - 1);
+    word_t x1 = digits[1] + (digits[0] >> Tr::word_shift), y1 = (y >> Tr::word_shift);
+    return x1 < y1 ? -1 : (x1 > y1 ? 1 : (x0 < y0 ? -1 : (x0 > y0 ? 1 : 0)));
+  } else {
+    return top_word() < 0 ? -1 : 1;
   }
 }
 
@@ -1312,17 +1323,14 @@ bool AnyIntView<Tr>::mod_div_any(const AnyIntView<Tr>& yp, AnyIntView<Tr>& quot,
       if (k > quot.max_size()) {
         return invalidate_bool();
       }
-      quot.set_size(max(k,1));
-      for(int qi=0; qi< max(k,1); qi++) {
-        quot.digits[qi]=0;
-      }
+      quot.set_size(std::max(k, 1));
+      quot.digits[0] = 0;
     } else {
       if (k >= quot.max_size()) {
         return invalidate_bool();
       }
       quot.set_size(k + 1);
-      double x_top = top_double();
-      word_t q = std::llrint(x_top * y_inv * Tr::InvBase);
+      word_t q = std::llrint(top_double() * y_inv * Tr::InvBase);
       quot.digits[k] = q;
       int i = yp.size() - 1;
       word_t hi = 0;
@@ -1337,24 +1345,26 @@ bool AnyIntView<Tr>::mod_div_any(const AnyIntView<Tr>& yp, AnyIntView<Tr>& quot,
     quot.digits[0] = 0;
   }
   while (--k >= 0) {
-    double x_top = top_double();
-    word_t q = std::llrint(x_top * y_inv);
+    word_t q = std::llrint(top_double() * y_inv);
     quot.digits[k] = q;
     for (int i = yp.size() - 1; i >= 0; --i) {
       Tr::sub_mul(&digits[k + i + 1], &digits[k + i], q, yp.digits[i]);
     }
     dec_size();
     digits[size() - 1] += (digits[size()] << word_shift);
   }
-  if (size() >= yp.size()) {
-    assert(size() == yp.size());
-    double x_top = top_double();
-    double t = x_top * y_inv * Tr::InvBase;
+  if (size() >= yp.size() - 1) {
+    assert(size() <= yp.size());
+    bool grow = (size() < yp.size());
+    double t = top_double() * y_inv * (grow ? Tr::InvBase * Tr::InvBase : Tr::InvBase);
     if (round_mode >= 0) {
       t += (round_mode ? 1 : 0.5);
     }
     word_t q = std::llrint(std::floor(t));
     if (q) {
+      if (grow) {
+        digits[inc_size()] = 0;
+      }
       for (int i = 0; i < size(); i++) {
         digits[i] -= q * yp.digits[i];
       }
@@ -1411,6 +1421,7 @@ bool AnyIntView<Tr>::mod_div_any(const AnyIntView<Tr>& yp, AnyIntView<Tr>& quot,
   return normalize_bool_any();
 }
 
+// works for almost-normalized numbers (digits -Base+1 .. Base-1, top non-zero), result also almost-normalized
 template <class Tr>
 bool AnyIntView<Tr>::mod_pow2_any(int exponent) {
   if (!is_valid()) {
@@ -1462,25 +1473,21 @@ bool AnyIntView<Tr>::mod_pow2_any(int exponent) {
     if (exponent >= max_size() * word_shift) {
       return invalidate_bool();
     }
-    if (q - word_shift >= 0) {
+    if (q - word_shift >= 0) {  // original top digit was a non-zero multiple of Base, impossible(?)
       digits[size() - 1] = 0;
       digits[inc_size()] = ((word_t)1 << (q - word_shift));
-    }
-    if (q - word_shift == -1 && size() < max_size() - 1) {
+    } else if (q - word_shift == -1 && size() < max_size()) {
       digits[size() - 1] = -Tr::Half;
       digits[inc_size()] = 1;
     } else {
       digits[size() - 1] = pow;
     }
     return true;
-  } else if (v >= Tr::Half) {
-    if (size() == max_size() - 1) {
-      return invalidate_bool();
-    } else {
-      digits[size() - 1] = v | -Tr::Half;
-      digits[inc_size()] = ((word_t)1 << (q - word_shift));
-      return true;
-    }
+  } else if (v >= Tr::Half && size() < max_size()) {
+    word_t w = (((v >> (word_shift - 1)) + 1) >> 1);
+    digits[size() - 1] = v - (w << word_shift);
+    digits[inc_size()] = w;
+    return true;
   } else {
     digits[size() - 1] = v;
     return true;