Merge pull request #697 from EmelyanenkoK/isolated_validate_improvement

Isolated validate improvement: higher auto validation limits and account for processed external messages when building block

crypto/block/block-parse.cpp

@@ -2292,5 +2292,37 @@ bool Aug_ShardFees::eval_leaf(vm::CellBuilder& cb, vm::CellSlice& cs) const {
 
 const Aug_ShardFees aug_ShardFees;
 
+bool validate_message_libs(const td::Ref<vm::Cell> &cell) {
+  gen::Message::Record rec;
+  if (!type_unpack_cell(cell, gen::t_Message_Any, rec)) {
+    return false;
+  }
+  vm::CellSlice& state_init = rec.init.write();
+  if (!state_init.fetch_long(1)) {
+    return true;
+  }
+  if (state_init.fetch_long(1)) {
+    return gen::t_StateInitWithLibs.validate_ref(state_init.prefetch_ref());
+  } else {
+    return gen::t_StateInitWithLibs.validate_csr(rec.init);
+  }
+}
+
+bool validate_message_relaxed_libs(const td::Ref<vm::Cell> &cell) {
+  gen::MessageRelaxed::Record rec;
+  if (!type_unpack_cell(cell, gen::t_MessageRelaxed_Any, rec)) {
+    return false;
+  }
+  vm::CellSlice& state_init = rec.init.write();
+  if (!state_init.fetch_long(1)) {
+    return true;
+  }
+  if (state_init.fetch_long(1)) {
+    return gen::t_StateInitWithLibs.validate_ref(state_init.prefetch_ref());
+  } else {
+    return gen::t_StateInitWithLibs.validate_csr(rec.init);
+  }
+}
+
 }  // namespace tlb
 }  // namespace block

crypto/block/block-parse.h

@@ -1113,5 +1113,9 @@ struct Aug_ShardFees final : AugmentationCheckData {
 
 extern const Aug_ShardFees aug_ShardFees;
 
+// Validate dict of libraries in message: used when sending and receiving message
+bool validate_message_libs(const td::Ref<vm::Cell> &cell);
+bool validate_message_relaxed_libs(const td::Ref<vm::Cell> &cell);
+
 }  // namespace tlb
 }  // namespace block

crypto/block/block.cpp

@@ -715,7 +715,7 @@ td::uint64 BlockLimitStatus::estimate_block_size(const vm::NewCellStorageStat::S
     sum += *extra;
   }
   return 2000 + (sum.bits >> 3) + sum.cells * 12 + sum.internal_refs * 3 + sum.external_refs * 40 + accounts * 200 +
-         transactions * 200 + (extra ? 200 : 0);
+         transactions * 200 + (extra ? 200 : 0) + extra_out_msgs * 300;
 }
 
 int BlockLimitStatus::classify() const {

crypto/block/block.h

@@ -261,7 +261,7 @@ struct BlockLimitStatus {
   ton::LogicalTime cur_lt;
   td::uint64 gas_used{};
   vm::NewCellStorageStat st_stat;
-  unsigned accounts{}, transactions{};
+  unsigned accounts{}, transactions{}, extra_out_msgs{};
   BlockLimitStatus(const BlockLimits& limits_, ton::LogicalTime lt = 0)
       : limits(limits_), cur_lt(std::max(limits_.start_lt, lt)) {
   }
@@ -270,6 +270,7 @@ struct BlockLimitStatus {
     st_stat.set_zero();
     transactions = accounts = 0;
     gas_used = 0;
+    extra_out_msgs = 0;
   }
   td::uint64 estimate_block_size(const vm::NewCellStorageStat::Stat* extra = nullptr) const;
   int classify() const;

crypto/block/block.tlb

@@ -143,8 +143,13 @@ tick_tock$_ tick:Bool tock:Bool = TickTock;
 
 _ split_depth:(Maybe (## 5)) special:(Maybe TickTock)
   code:(Maybe ^Cell) data:(Maybe ^Cell)
-  library:(HashmapE 256 SimpleLib) = StateInit;
-
+  library:(Maybe ^Cell) = StateInit;
+
+// StateInitWithLibs is used to validate sent and received messages
+_ split_depth:(Maybe (## 5)) special:(Maybe TickTock)
+  code:(Maybe ^Cell) data:(Maybe ^Cell)
+  library:(HashmapE 256 SimpleLib) = StateInitWithLibs;
+
 simple_lib$_ public:Bool root:^Cell = SimpleLib;
 
 message$_ {X:Type} info:CommonMsgInfo

crypto/block/transaction.cpp

@@ -635,14 +635,15 @@ bool Transaction::unpack_input_msg(bool ihr_delivered, const ActionPhaseConfig*
       vm::CellBuilder cb;
       if (!(cs.advance(2) && block::gen::t_StateInit.fetch_to(cs, state_init) &&
             cb.append_cellslice_bool(std::move(state_init)) && cb.finalize_to(in_msg_state) &&
-            block::gen::t_StateInit.validate_ref(in_msg_state))) {
+            block::gen::t_StateInitWithLibs.validate_ref(in_msg_state))) {
         LOG(DEBUG) << "cannot parse StateInit in inbound message";
         return false;
       }
       break;
     }
     case 3: {  // (just$1 (right$1 _:^StateInit ))
-      if (!(cs.advance(2) && cs.fetch_ref_to(in_msg_state) && block::gen::t_StateInit.validate_ref(in_msg_state))) {
+      if (!(cs.advance(2) && cs.fetch_ref_to(in_msg_state) &&
+            block::gen::t_StateInitWithLibs.validate_ref(in_msg_state))) {
         LOG(DEBUG) << "cannot parse ^StateInit in inbound message";
         return false;
       }
@@ -1534,6 +1535,10 @@ int Transaction::try_action_send_msg(const vm::CellSlice& cs0, ActionPhase& ap,
   if (!tlb::type_unpack_cell(act_rec.out_msg, block::gen::t_MessageRelaxed_Any, msg)) {
     return -1;
   }
+  if (!block::tlb::validate_message_relaxed_libs(act_rec.out_msg)) {
+    LOG(DEBUG) << "outbound message has invalid libs in StateInit";
+    return -1;
+  }
   if (redoing >= 1) {
     if (msg.init->size_refs() >= 2) {
       LOG(DEBUG) << "moving the StateInit of a suggested outbound message into a separate cell";

validator/impl/collator.cpp

@@ -2565,7 +2565,7 @@ bool Collator::process_inbound_message(Ref<vm::CellSlice> enq_msg, ton::LogicalT
     return false;
   }
   if (!block::tlb::t_MsgEnvelope.validate_ref(msg_env)) {
-    LOG(ERROR) << "inbound internal MsgEnvelope is invalid according to automated checks";
+    LOG(ERROR) << "inbound internal MsgEnvelope is invalid according to hand-written checks";
     return false;
   }
   // 1. unpack MsgEnvelope
@@ -2590,6 +2590,10 @@ bool Collator::process_inbound_message(Ref<vm::CellSlice> enq_msg, ton::LogicalT
                   "its contents";
     return false;
   }
+  if (!block::tlb::validate_message_libs(env.msg)) {
+    LOG(ERROR) << "inbound internal message has invalid StateInit";
+    return false;
+  }
   // 2.0. update last_proc_int_msg
   if (!update_last_proc_int_msg(std::pair<ton::LogicalTime, ton::Bits256>(lt, env.msg->get_hash().bits()))) {
     return fatal_error("processing a message AFTER a newer message has been processed");
@@ -2957,6 +2961,7 @@ bool Collator::process_new_messages(bool enqueue_only) {
   while (!new_msgs.empty()) {
     block::NewOutMsg msg = new_msgs.top();
     new_msgs.pop();
+    block_limit_status_->extra_out_msgs--;
     if (block_full_ && !enqueue_only) {
       LOG(INFO) << "BLOCK FULL, enqueue all remaining new messages";
       enqueue_only = true;
@@ -2978,6 +2983,7 @@ void Collator::register_new_msg(block::NewOutMsg new_msg) {
     min_new_msg_lt = new_msg.lt;
   }
   new_msgs.push(std::move(new_msg));
+  block_limit_status_->extra_out_msgs++;
 }
 
 void Collator::register_new_msgs(block::transaction::Transaction& trans) {
@@ -3876,7 +3882,7 @@ bool Collator::create_block() {
   }
   if (verify >= 1) {
     LOG(INFO) << "verifying new Block";
-    if (!block::gen::t_Block.validate_ref(1000000, new_block)) {
+    if (!block::gen::t_Block.validate_ref(10000000, new_block)) {
       return fatal_error("new Block failed to pass automatic validity tests");
     }
   }
@@ -3968,6 +3974,18 @@ bool Collator::create_block_candidate() {
       ton::BlockIdExt{ton::BlockId{shard_, new_block_seqno}, new_block->get_hash().bits(),
                       block::compute_file_hash(blk_slice.as_slice())},
       block::compute_file_hash(cdata_slice.as_slice()), blk_slice.clone(), cdata_slice.clone());
+  // 3.1 check block and collated data size
+  auto consensus_config = config_->get_consensus_config();
+  if (block_candidate->data.size() > consensus_config.max_block_size) {
+    return fatal_error(PSTRING() << "block size (" << block_candidate->data.size()
+                                 << ") exceeds the limit in consensus config (" << consensus_config.max_block_size
+                                 << ")");
+  }
+  if (block_candidate->collated_data.size() > consensus_config.max_collated_data_size) {
+    return fatal_error(PSTRING() << "collated data size (" << block_candidate->collated_data.size()
+                                 << ") exceeds the limit in consensus config ("
+                                 << consensus_config.max_collated_data_size << ")");
+  }
   // 4. save block candidate
   LOG(INFO) << "saving new BlockCandidate";
   td::actor::send_closure_later(manager, &ValidatorManager::set_block_candidate, block_candidate->id,
@@ -4030,6 +4048,9 @@ td::Result<bool> Collator::register_external_message_cell(Ref<vm::Cell> ext_msg,
   if (!block::tlb::t_Message.validate_ref(256, ext_msg)) {
     return td::Status::Error("external message is not a (Message Any) according to hand-written checks");
   }
+  if (!block::tlb::validate_message_libs(ext_msg)) {
+    return td::Status::Error("external message has invalid libs in StateInit");
+  }
   block::gen::CommonMsgInfo::Record_ext_in_msg_info info;
   if (!tlb::unpack_cell_inexact(ext_msg, info)) {
     return td::Status::Error("cannot unpack external message header");

validator/impl/validate-query.cpp

@@ -2111,13 +2111,13 @@ bool ValidateQuery::unpack_block_data() {
   auto outmsg_cs = vm::load_cell_slice_ref(std::move(extra.out_msg_descr));
   // run some hand-written checks from block::tlb::
   // (automatic tests from block::gen:: have been already run for the entire block)
-  if (!block::tlb::t_InMsgDescr.validate_upto(1000000, *inmsg_cs)) {
+  if (!block::tlb::t_InMsgDescr.validate_upto(10000000, *inmsg_cs)) {
     return reject_query("InMsgDescr of the new block failed to pass handwritten validity tests");
   }
-  if (!block::tlb::t_OutMsgDescr.validate_upto(1000000, *outmsg_cs)) {
+  if (!block::tlb::t_OutMsgDescr.validate_upto(10000000, *outmsg_cs)) {
     return reject_query("OutMsgDescr of the new block failed to pass handwritten validity tests");
   }
-  if (!block::tlb::t_ShardAccountBlocks.validate_ref(1000000, extra.account_blocks)) {
+  if (!block::tlb::t_ShardAccountBlocks.validate_ref(10000000, extra.account_blocks)) {
     return reject_query("ShardAccountBlocks of the new block failed to pass handwritten validity tests");
   }
   in_msg_dict_ = std::make_unique<vm::AugmentedDictionary>(std::move(inmsg_cs), 256, block::tlb::aug_InMsgDescr);
@@ -5507,7 +5507,7 @@ bool ValidateQuery::try_validate() {
       }
     }
     LOG(INFO) << "running automated validity checks for block candidate " << id_.to_str();
-    if (!block::gen::t_Block.validate_ref(1000000, block_root_)) {
+    if (!block::gen::t_Block.validate_ref(10000000, block_root_)) {
       return reject_query("block "s + id_.to_str() + " failed to pass automated validity checks");
     }
     if (!fix_all_processed_upto()) {