From fd4c512de51251b8a5cbd62f3d2cb24063292e42 Mon Sep 17 00:00:00 2001 From: EmelyanenkoK Date: Mon, 17 Apr 2023 15:54:33 +0300 Subject: [PATCH 01/20] Move JettonBridge config num for BNB, Polygon --- crypto/block/block.tlb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crypto/block/block.tlb b/crypto/block/block.tlb index 8662e2434..0c8fdb83e 100644 --- a/crypto/block/block.tlb +++ b/crypto/block/block.tlb @@ -778,8 +778,8 @@ jetton_bridge_params_v0#00 bridge_address:bits256 oracles_address:bits256 oracle jetton_bridge_params_v1#01 bridge_address:bits256 oracles_address:bits256 oracles:(HashmapE 256 uint256) state_flags:uint8 prices:^JettonBridgePrices external_chain_address:bits256 = JettonBridgeParams; _ JettonBridgeParams = ConfigParam 79; // ETH->TON token bridge -_ JettonBridgeParams = ConfigParam 80; // BNB->TON token bridge -_ JettonBridgeParams = ConfigParam 81; // Polygon->TON token bridge +_ JettonBridgeParams = ConfigParam 81; // BNB->TON token bridge +_ JettonBridgeParams = ConfigParam 82; // Polygon->TON token bridge // From 8d919a5db9af1af66ac6475ce28249e157a5deb2 Mon Sep 17 00:00:00 2001 From: Ilyar <761285+ilyar@users.noreply.github.com> Date: Wed, 19 Apr 2023 19:09:15 +0200 Subject: [PATCH 02/20] add missing VM opcode definition (#670) --- crypto/fift/lib/Asm.fif | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/crypto/fift/lib/Asm.fif b/crypto/fift/lib/Asm.fif index 85045f290..937358e08 100644 --- a/crypto/fift/lib/Asm.fif +++ b/crypto/fift/lib/Asm.fif @@ -1194,6 +1194,11 @@ x{FA01} @Defop LDVARINT16 x{FA02} dup @Defop STGRAMS @Defop STVARUINT16 x{FA03} @Defop STVARINT16 +x{FA04} @Defop LDVARUINT32 // (s -- x s') +x{FA05} @Defop LDVARINT32 // (s -- x s') +x{FA06} @Defop STVARUINT32 // (b x -- b') +x{FA07} @Defop STVARINT32 // (b x -- b') + x{FA40} @Defop LDMSGADDR x{FA41} @Defop LDMSGADDRQ x{FA42} @Defop PARSEMSGADDR From 8b0d6a2665cb8049dbe08790172874143a07db8a Mon Sep 17 00:00:00 2001 From: EmelyanenkoK Date: Wed, 19 Apr 2023 21:29:41 +0300 Subject: [PATCH 03/20] Add WASM FunC autotests (#673) * feat: func wasm autotests * fixes necessary for func wasm autotests --------- Co-authored-by: krigga --- crypto/func/asmops.cpp | 6 +- crypto/func/auto-tests/legacy_tester.js | 27 ++++++++ crypto/func/auto-tests/legacy_tests.json | 1 + crypto/func/auto-tests/run_tests.js | 77 +++++++++++++++++++++ crypto/func/auto-tests/wasm_tests_common.js | 63 +++++++++++++++++ crypto/funcfiftlib/funcfiftlib.cpp | 1 + 6 files changed, 172 insertions(+), 3 deletions(-) create mode 100644 crypto/func/auto-tests/legacy_tester.js create mode 100644 crypto/func/auto-tests/legacy_tests.json create mode 100644 crypto/func/auto-tests/run_tests.js create mode 100644 crypto/func/auto-tests/wasm_tests_common.js diff --git a/crypto/func/asmops.cpp b/crypto/func/asmops.cpp index ccc409d39..71ee58f68 100644 --- a/crypto/func/asmops.cpp +++ b/crypto/func/asmops.cpp @@ -166,7 +166,7 @@ AsmOp AsmOp::UnTuple(int a) { AsmOp AsmOp::IntConst(td::RefInt256 x) { if (x->signed_fits_bits(8)) { - return AsmOp::Const(dec_string(std::move(x)) + " PUSHINT", x); + return AsmOp::Const(dec_string(x) + " PUSHINT", x); } if (!x->is_valid()) { return AsmOp::Const("PUSHNAN", x); @@ -184,9 +184,9 @@ AsmOp AsmOp::IntConst(td::RefInt256 x) { return AsmOp::Const(k, "PUSHNEGPOW2", x); } if (!x->mod_pow2_short(23)) { - return AsmOp::Const(dec_string(std::move(x)) + " PUSHINTX", x); + return AsmOp::Const(dec_string(x) + " PUSHINTX", x); } - return AsmOp::Const(dec_string(std::move(x)) + " PUSHINT", x); + return AsmOp::Const(dec_string(x) + " PUSHINT", x); } AsmOp AsmOp::BoolConst(bool f) { diff --git a/crypto/func/auto-tests/legacy_tester.js b/crypto/func/auto-tests/legacy_tester.js new file mode 100644 index 000000000..57092d683 --- /dev/null +++ b/crypto/func/auto-tests/legacy_tester.js @@ -0,0 +1,27 @@ +const fs = require('fs/promises'); +const { compileWasm, compileFile } = require('./wasm_tests_common'); + +async function main() { + const tests = JSON.parse((await fs.readFile('../legacy_tests.json')).toString('utf-8')) + + for (const [filename, hashstr] of tests) { + if (filename.includes('storage-provider')) continue; + + const mod = await compileWasm() + + const response = await compileFile(mod, filename); + + if (response.status !== 'ok') { + console.error(response); + throw new Error('Could not compile ' + filename); + } + + if (BigInt('0x' + response.codeHashHex) !== BigInt(hashstr)) { + throw new Error('Compilation result is different for ' + filename); + } + + console.log(filename, 'ok'); + } +} + +main() \ No newline at end of file diff --git a/crypto/func/auto-tests/legacy_tests.json b/crypto/func/auto-tests/legacy_tests.json new file mode 100644 index 000000000..61a433bb7 --- /dev/null +++ b/crypto/func/auto-tests/legacy_tests.json @@ -0,0 +1 @@ +[["elector/elector-code.fc", "115226404411715505328583639896096915745686314074575650766750648324043316883483"], ["config/config-code.fc", "10913070768607625342121305745084703121685937915388357634624451844356456145601"], ["eth-bridge-multisig/multisig-code.fc", "101509909129354488841890823627011033360100627957439967918234053299675481277954"], ["bsc-bridge-collector/votes-collector.fc", "62190447221288642706570413295807615918589884489514159926097051017036969900417"], ["uni-lock-wallet/uni-lockup-wallet.fc", "61959738324779104851267145467044677651344601417998258530238254441977103654381"], ["nft-collection/nft-collection-editable.fc", "45561997735512210616567774035540357815786262097548276229169737015839077731274"], ["dns-collection/nft-collection.fc", "107999822699841936063083742021519765435859194241091312445235370766165379261859"], ["tele-nft-item/nft-item.fc", "69777543125381987786450436977742010705076866061362104025338034583422166453344"], ["storage/storage-contract.fc", "91377830060355733016937375216020277778264560226873154627574229667513068328151"], ["storage/storage-provider.fc", "13618336676213331164384407184540461509022654507176709588621016553953760588122"], ["nominator-pool/pool.fc", "69767057279163099864792356875696330339149706521019810113334238732928422055375"], ["jetton-minter/jetton-minter.fc", "9028309926287301331466371999814928201427184114165428257502393474125007156494"], ["gg-marketplace/nft-marketplace-v2.fc", "92199806964112524639740773542356508485601908152150843819273107618799016205930"], ["jetton-wallet/jetton-wallet.fc", "86251125787443633057458168028617933212663498001665054651523310772884328206542"], ["whales-nominators/nominators.fc", "8941364499854379927692172316865293429893094891593442801401542636695127885153"], ["tact-examples/treasure_Treasure.code.fc", "13962538639825790677138656603323869918938565499584297120566680287245364723897"], ["tact-examples/jetton_SampleJetton.code.fc", "94076762218493729104783735200107713211245710256802265203823917715299139499110"], ["tact-examples/jetton_JettonDefaultWallet.code.fc", "29421313492520031238091587108198906058157443241743283101866538036369069620563"], ["tact-examples/maps_MapTestContract.code.fc", "22556550222249123835909180266811414538971143565993192846012583552876721649744"]] \ No newline at end of file diff --git a/crypto/func/auto-tests/run_tests.js b/crypto/func/auto-tests/run_tests.js new file mode 100644 index 000000000..f8e6c6a78 --- /dev/null +++ b/crypto/func/auto-tests/run_tests.js @@ -0,0 +1,77 @@ +const fs = require('fs/promises'); +const os = require('os'); +const path = require('path'); +const { compileWasm, compileFile } = require('./wasm_tests_common'); +const { execSync } = require('child_process'); + +async function main() { + const compiledPath = path.join(os.tmpdir(), 'compiled.fif'); + const runnerPath = path.join(os.tmpdir(), 'runner.fif'); + + const tests = (await fs.readdir('.')).filter(f => f.endsWith('.fc')).sort(); + + const mathChars = '0x123456789()+-*/<>'.split('') + + for (const testFile of tests) { + const mod = await compileWasm() + + const result = await compileFile(mod, testFile) + + if (result.status !== 'ok') { + console.error(result); + throw new Error('Could not compile ' + filename); + } + + const fileLines = (await fs.readFile(testFile)).toString('utf-8').split('\n'); + + const testCases = []; + + for (const line of fileLines) { + const parts = line.split('|').map(c => c.trim()); + + if (parts.length !== 4 || parts[0] !== 'TESTCASE') continue; + + const processedInputs = []; + + for (const input of parts[2].split(' ')) { + if (input.includes('x{')) { + processedInputs.push(input); + continue; + } + + if (input.length === 0) { + continue + } + + const replacedInput = input.split('').filter(c => mathChars.includes(c)).join('').replace('//', '/').replace(/([0-9a-f])($|[^0-9a-fx])/gmi, '$1n$2') + + processedInputs.push(eval(replacedInput).toString()); + } + + testCases.push([parts[1], processedInputs.join(' '), parts[3]]); + } + + await fs.writeFile(compiledPath, '"Asm.fif" include\n' + JSON.parse('"' + result.fiftCode + '"')); + await fs.writeFile(runnerPath, `"${compiledPath}" include `${t[1]} ${t[0]} code 1 runvmx abort"exitcode is not 0" .s cr { drop } depth 1- times`).join('\n')}`) + + const fiftResult = execSync(`${process.env.FIFT_EXECUTABLE || 'fift'} -I ${process.env.FIFT_LIBS} /tmp/runner.fif`, { + stdio: ['pipe', 'pipe', 'ignore'] + }).toString('utf-8') + + const testResults = fiftResult.split('\n').map(s => s.trim()).filter(s => s.length > 0) + + if (testResults.length !== testCases.length) { + throw new Error(`Got ${testResults.length} results but there are ${testCases.length} cases`) + } + + for (let i = 0; i < testResults.length; i++) { + if (testResults[i] !== testCases[i][2]) { + throw new Error(`Unequal result ${testResults[i]} and case ${testCases[i][2]}`) + } + } + + console.log(testFile, 'ok') + } +} + +main() \ No newline at end of file diff --git a/crypto/func/auto-tests/wasm_tests_common.js b/crypto/func/auto-tests/wasm_tests_common.js new file mode 100644 index 000000000..84d8e0d57 --- /dev/null +++ b/crypto/func/auto-tests/wasm_tests_common.js @@ -0,0 +1,63 @@ +const fsSync = require('fs'); + +const copyToCString = (mod, str) => { + const len = mod.lengthBytesUTF8(str) + 1; + const ptr = mod._malloc(len); + mod.stringToUTF8(str, ptr, len); + return ptr; +}; + +const copyToCStringPtr = (mod, str, ptr) => { + const allocated = copyToCString(mod, str); + mod.setValue(ptr, allocated, '*'); + return allocated; +}; + +const copyFromCString = (mod, ptr) => { + return mod.UTF8ToString(ptr); +}; + +async function compileFile(mod, filename) { + const callbackPtr = mod.addFunction((_kind, _data, contents, error) => { + const kind = copyFromCString(mod, _kind); + const data = copyFromCString(mod, _data); + if (kind === 'realpath') { + copyToCStringPtr(mod, fsSync.realpathSync(data), contents); + } else if (kind === 'source') { + const path = fsSync.realpathSync(data); + try { + copyToCStringPtr(mod, fsSync.readFileSync(path).toString('utf-8'), contents); + } catch (err) { + copyToCStringPtr(mod, e.message, error); + } + } else { + copyToCStringPtr(mod, 'Unknown callback kind ' + kind, error); + } + }, 'viiii'); + + const config = { + optLevel: 2, + sources: [filename] + }; + + const configPtr = copyToCString(mod, JSON.stringify(config)); + + const responsePtr = mod._func_compile(configPtr, callbackPtr); + + return JSON.parse(copyFromCString(mod, responsePtr)); +} + +const wasmModule = require(process.env.FUNCFIFTLIB_MODULE) + +const wasmBinary = new Uint8Array(fsSync.readFileSync(process.env.FUNCFIFTLIB_WASM)) + +async function compileWasm() { + const mod = await wasmModule({ wasmBinary }) + + return mod +} + +module.exports = { + compileFile, + compileWasm +} diff --git a/crypto/funcfiftlib/funcfiftlib.cpp b/crypto/funcfiftlib/funcfiftlib.cpp index 070c3e0d2..c8bf4fc56 100644 --- a/crypto/funcfiftlib/funcfiftlib.cpp +++ b/crypto/funcfiftlib/funcfiftlib.cpp @@ -92,6 +92,7 @@ td::Result compile_internal(char *config_json) { result_obj("status", "ok"); result_obj("codeBoc", td::base64_encode(boc)); result_obj("fiftCode", escape_json(outs.str())); + result_obj("codeHashHex", code_cell->get_hash().to_hex()); result_obj.leave(); outs.clear(); From 7ee70778d2f2d2e06d83a545ec83e9c09267ef79 Mon Sep 17 00:00:00 2001 From: EmelyanenkoK Date: Wed, 26 Apr 2023 09:15:16 +0300 Subject: [PATCH 04/20] Add timeout for out_msg_queue_cleanup (#679) Co-authored-by: SpyCheese --- validator/impl/collator-impl.h | 2 +- validator/impl/collator.cpp | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/validator/impl/collator-impl.h b/validator/impl/collator-impl.h index dfe844a32..7b136953d 100644 --- a/validator/impl/collator-impl.h +++ b/validator/impl/collator-impl.h @@ -73,7 +73,7 @@ class Collator final : public td::actor::Actor { Ref validator_set_; td::actor::ActorId manager; td::Timestamp timeout; - td::Timestamp soft_timeout_, medium_timeout_; + td::Timestamp queue_cleanup_timeout_, soft_timeout_, medium_timeout_; td::Promise main_promise; ton::BlockSeqno last_block_seqno{0}; ton::BlockSeqno prev_mc_block_seqno{0}; diff --git a/validator/impl/collator.cpp b/validator/impl/collator.cpp index 83bc710ce..6debf2de5 100644 --- a/validator/impl/collator.cpp +++ b/validator/impl/collator.cpp @@ -67,6 +67,7 @@ Collator::Collator(ShardIdFull shard, bool is_hardfork, UnixTime min_ts, BlockId , validator_set_(std::move(validator_set)) , manager(manager) , timeout(timeout) + , queue_cleanup_timeout_(td::Timestamp::at(timeout.at() - 5.0)) , soft_timeout_(td::Timestamp::at(timeout.at() - 3.0)) , medium_timeout_(td::Timestamp::at(timeout.at() - 1.5)) , main_promise(std::move(promise)) @@ -1814,6 +1815,11 @@ bool Collator::out_msg_queue_cleanup() { auto res = out_msg_queue_->filter([&](vm::CellSlice& cs, td::ConstBitPtr key, int n) -> int { assert(n == 352); // LOG(DEBUG) << "key is " << key.to_hex(n); + if (queue_cleanup_timeout_.is_in_past(td::Timestamp::now())) { + LOG(WARNING) << "cleaning up outbound queue takes too long, ending"; + outq_cleanup_partial_ = true; + return (1 << 30) + 1; // retain all remaining outbound queue entries including this one without processing + } if (block_full_) { LOG(WARNING) << "BLOCK FULL while cleaning up outbound queue, cleanup completed only partially"; outq_cleanup_partial_ = true; From d067e6ca3fa6a694905b1a7c93bb1748bd287195 Mon Sep 17 00:00:00 2001 From: EmelyanenkoK Date: Wed, 26 Apr 2023 21:32:26 +0300 Subject: [PATCH 05/20] Make stricter cleanup timelimit --- validator/impl/collator.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/validator/impl/collator.cpp b/validator/impl/collator.cpp index 6debf2de5..6935f83c2 100644 --- a/validator/impl/collator.cpp +++ b/validator/impl/collator.cpp @@ -67,7 +67,7 @@ Collator::Collator(ShardIdFull shard, bool is_hardfork, UnixTime min_ts, BlockId , validator_set_(std::move(validator_set)) , manager(manager) , timeout(timeout) - , queue_cleanup_timeout_(td::Timestamp::at(timeout.at() - 5.0)) + , queue_cleanup_timeout_(td::Timestamp::at(timeout.at() - 9.0)) , soft_timeout_(td::Timestamp::at(timeout.at() - 3.0)) , medium_timeout_(td::Timestamp::at(timeout.at() - 1.5)) , main_promise(std::move(promise)) From aab1fe0751a9bcddf076144f387db732b36c32af Mon Sep 17 00:00:00 2001 From: "aleksej.paschenko" Date: Thu, 27 Apr 2023 09:33:32 +0300 Subject: [PATCH 06/20] Use BUILD_SHARED_LIBS to decide whether to build libemulator.so (#671) * Update ubuntu:20.04 dockerfile (#636) * Update Dockerfile * Update Dockerfile --------- Co-authored-by: neodiX42 * Use BUILD_SHARED_LIBS to decide whether to build libemulator.so --------- Co-authored-by: EmelyanenkoK Co-authored-by: neodiX42 --- emulator/CMakeLists.txt | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/emulator/CMakeLists.txt b/emulator/CMakeLists.txt index 4e67d32cc..1389050c6 100644 --- a/emulator/CMakeLists.txt +++ b/emulator/CMakeLists.txt @@ -1,5 +1,7 @@ cmake_minimum_required(VERSION 3.0.2 FATAL_ERROR) +option(BUILD_SHARED_LIBS "Use \"OFF\" for a static build." ON) + if (NOT OPENSSL_FOUND) find_package(OpenSSL REQUIRED) endif() @@ -27,7 +29,12 @@ include(GenerateExportHeader) add_library(emulator_static STATIC ${EMULATOR_STATIC_SOURCE}) target_link_libraries(emulator_static PUBLIC ton_crypto ton_block smc-envelope) -add_library(emulator STATIC ${EMULATOR_SOURCE} ${EMULATOR_HEADERS}) +if (NOT USE_EMSCRIPTEN AND BUILD_SHARED_LIBS) + add_library(emulator SHARED ${EMULATOR_SOURCE} ${EMULATOR_HEADERS}) +else() + add_library(emulator STATIC ${EMULATOR_SOURCE} ${EMULATOR_HEADERS}) +endif() + target_link_libraries(emulator PUBLIC emulator_static) generate_export_header(emulator EXPORT_FILE_NAME ${CMAKE_CURRENT_BINARY_DIR}/emulator_export.h) target_include_directories(emulator PUBLIC From 5606418234d1074ce523c76da1a9ac9bfc760c2a Mon Sep 17 00:00:00 2001 From: EmelyanenkoK Date: Thu, 27 Apr 2023 10:23:04 +0300 Subject: [PATCH 07/20] FunC: Prohibit unifying tensors and "forall" vars (#684) * FunC: Prohibit unifying tensors and "forall" vars * Bump funC version to 0.4.4 --------- Co-authored-by: SpyCheese --- crypto/func/func.h | 5 +++-- crypto/func/unify-types.cpp | 31 +++++++++++++++++++++---------- 2 files changed, 24 insertions(+), 12 deletions(-) diff --git a/crypto/func/func.h b/crypto/func/func.h index 10039ffae..33f8c86f0 100644 --- a/crypto/func/func.h +++ b/crypto/func/func.h @@ -41,7 +41,7 @@ extern std::string generated_from; constexpr int optimize_depth = 20; -const std::string func_version{"0.4.3"}; +const std::string func_version{"0.4.4"}; enum Keyword { _Eof = -1, @@ -159,6 +159,7 @@ struct TypeExpr { int minw, maxw; static constexpr int w_inf = 1023; std::vector args; + bool was_forall_var = false; TypeExpr(te_type _constr, int _val = 0) : constr(_constr), value(_val), minw(0), maxw(w_inf) { } TypeExpr(te_type _constr, int _val, int width) : constr(_constr), value(_val), minw(width), maxw(width) { @@ -265,7 +266,7 @@ struct TypeExpr { return new TypeExpr{te_ForAll, body, std::move(list)}; } static bool remove_indirect(TypeExpr*& te, TypeExpr* forbidden = nullptr); - static bool remove_forall(TypeExpr*& te); + static std::vector remove_forall(TypeExpr*& te); static bool remove_forall_in(TypeExpr*& te, TypeExpr* te2, const std::vector& new_vars); }; diff --git a/crypto/func/unify-types.cpp b/crypto/func/unify-types.cpp index 517299e90..dfa1f6029 100644 --- a/crypto/func/unify-types.cpp +++ b/crypto/func/unify-types.cpp @@ -146,11 +146,8 @@ bool TypeExpr::remove_indirect(TypeExpr*& te, TypeExpr* forbidden) { return res; } -bool TypeExpr::remove_forall(TypeExpr*& te) { - assert(te); - if (te->constr != te_ForAll) { - return false; - } +std::vector TypeExpr::remove_forall(TypeExpr*& te) { + assert(te && te->constr == te_ForAll); assert(te->args.size() >= 1); std::vector new_vars; for (std::size_t i = 1; i < te->args.size(); i++) { @@ -161,7 +158,7 @@ bool TypeExpr::remove_forall(TypeExpr*& te) { te = te->args[0]; remove_forall_in(te, te2, new_vars); // std::cerr << "-> " << te << std::endl; - return true; + return new_vars; } bool TypeExpr::remove_forall_in(TypeExpr*& te, TypeExpr* te2, const std::vector& new_vars) { @@ -363,20 +360,34 @@ void unify(TypeExpr*& te1, TypeExpr*& te2) { } if (te1->constr == TypeExpr::te_ForAll) { TypeExpr* te = te1; - if (!TypeExpr::remove_forall(te)) { - throw UnifyError{te1, te2, "cannot remove universal type quantifier while performing type unification"}; + std::vector new_vars = TypeExpr::remove_forall(te); + for (TypeExpr* t : new_vars) { + t->was_forall_var = true; } unify(te, te2); + for (TypeExpr* t : new_vars) { + t->was_forall_var = false; + } return; } if (te2->constr == TypeExpr::te_ForAll) { TypeExpr* te = te2; - if (!TypeExpr::remove_forall(te)) { - throw UnifyError{te2, te1, "cannot remove universal type quantifier while performing type unification"}; + std::vector new_vars = TypeExpr::remove_forall(te); + for (TypeExpr* t : new_vars) { + t->was_forall_var = true; } unify(te1, te); + for (TypeExpr* t : new_vars) { + t->was_forall_var = false; + } return; } + if (te1->was_forall_var && te2->constr == TypeExpr::te_Tensor) { + throw UnifyError{te1, te2, "cannot unify generic type and tensor"}; + } + if (te2->was_forall_var && te1->constr == TypeExpr::te_Tensor) { + throw UnifyError{te2, te1, "cannot unify generic type and tensor"}; + } if (te1->constr == TypeExpr::te_Unknown) { if (te2->constr == TypeExpr::te_Unknown) { assert(te1->value != te2->value); From dd8658c4db2e3975e61f9aee6db59a049d79ac97 Mon Sep 17 00:00:00 2001 From: EmelyanenkoK Date: Thu, 27 Apr 2023 14:43:37 +0300 Subject: [PATCH 08/20] Comment out excessive checks of outbound messages queue --- validator/impl/collator.cpp | 1 + validator/impl/validate-query.cpp | 5 +++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/validator/impl/collator.cpp b/validator/impl/collator.cpp index 6935f83c2..1c3cafe6b 100644 --- a/validator/impl/collator.cpp +++ b/validator/impl/collator.cpp @@ -67,6 +67,7 @@ Collator::Collator(ShardIdFull shard, bool is_hardfork, UnixTime min_ts, BlockId , validator_set_(std::move(validator_set)) , manager(manager) , timeout(timeout) + // default timeout is 10 seconds, declared in validator/validator-group.cpp:generate_block_candidate:run_collate_query , queue_cleanup_timeout_(td::Timestamp::at(timeout.at() - 9.0)) , soft_timeout_(td::Timestamp::at(timeout.at() - 3.0)) , medium_timeout_(td::Timestamp::at(timeout.at() - 1.5)) diff --git a/validator/impl/validate-query.cpp b/validator/impl/validate-query.cpp index 7d8104e06..0e1b11998 100644 --- a/validator/impl/validate-query.cpp +++ b/validator/impl/validate-query.cpp @@ -5540,9 +5540,10 @@ bool ValidateQuery::try_validate() { if (!check_in_queue()) { return reject_query("cannot check inbound message queues"); } - if (!check_delivered_dequeued()) { + // Excessive check: validity of message in queue is checked elsewhere + /*if (!check_delivered_dequeued()) { return reject_query("cannot check delivery status of all outbound messages"); - } + }*/ if (!check_transactions()) { return reject_query("invalid collection of account transactions in ShardAccountBlocks"); } From 8bc20ede2e148aa08444fa415fc7c5f15eb4ef0d Mon Sep 17 00:00:00 2001 From: EmelyanenkoK Date: Fri, 28 Apr 2023 12:58:02 +0300 Subject: [PATCH 09/20] Add archive manager index (#685) * Optimize get_file_desc_by_ seqno/lt/ut * Optimize get_next_file_desc --------- Co-authored-by: SpyCheese --- validator/db/archive-manager.cpp | 333 +++++++++++++++++-------------- validator/db/archive-manager.hpp | 103 +++++++--- 2 files changed, 260 insertions(+), 176 deletions(-) diff --git a/validator/db/archive-manager.cpp b/validator/db/archive-manager.cpp index bd239bfc4..7c2aefe3e 100644 --- a/validator/db/archive-manager.cpp +++ b/validator/db/archive-manager.cpp @@ -74,7 +74,7 @@ void ArchiveManager::add_handle(BlockHandle handle, td::Promise promis } void ArchiveManager::update_handle(BlockHandle handle, td::Promise promise) { - FileDescription *f; + const FileDescription *f; if (handle->handle_moved_to_archive()) { CHECK(handle->inited_unix_time()); if (!handle->need_flush()) { @@ -439,15 +439,15 @@ void ArchiveManager::check_persistent_state(BlockIdExt block_id, BlockIdExt mast void ArchiveManager::get_block_by_unix_time(AccountIdPrefixFull account_id, UnixTime ts, td::Promise promise) { - auto f = get_file_desc_by_unix_time(account_id, ts, false); - if (f) { - auto n = f; - do { - n = get_next_file_desc(n); - } while (n != nullptr && !n->has_account_prefix(account_id)); + auto f1 = get_file_desc_by_unix_time(account_id, ts, false); + auto f2 = get_next_file_desc(f1, account_id, false); + if (!f1) { + std::swap(f1, f2); + } + if (f1) { td::actor::ActorId aid; - if (n) { - aid = n->file_actor_id(); + if (f2) { + aid = f2->file_actor_id(); } auto P = td::PromiseCreator::lambda( [aid, account_id, ts, promise = std::move(promise)](td::Result R) mutable { @@ -457,7 +457,7 @@ void ArchiveManager::get_block_by_unix_time(AccountIdPrefixFull account_id, Unix td::actor::send_closure(aid, &ArchiveSlice::get_block_by_unix_time, account_id, ts, std::move(promise)); } }); - td::actor::send_closure(f->file_actor_id(), &ArchiveSlice::get_block_by_unix_time, account_id, ts, std::move(P)); + td::actor::send_closure(f1->file_actor_id(), &ArchiveSlice::get_block_by_unix_time, account_id, ts, std::move(P)); } else { promise.set_error(td::Status::Error(ErrorCode::notready, "ts not in db")); } @@ -465,15 +465,15 @@ void ArchiveManager::get_block_by_unix_time(AccountIdPrefixFull account_id, Unix void ArchiveManager::get_block_by_lt(AccountIdPrefixFull account_id, LogicalTime lt, td::Promise promise) { - auto f = get_file_desc_by_lt(account_id, lt, false); - if (f) { - auto n = f; - do { - n = get_next_file_desc(n); - } while (n != nullptr && !n->has_account_prefix(account_id)); + auto f1 = get_file_desc_by_lt(account_id, lt, false); + auto f2 = get_next_file_desc(f1, account_id, false); + if (!f1) { + std::swap(f1, f2); + } + if (f1) { td::actor::ActorId aid; - if (n) { - aid = n->file_actor_id(); + if (f2) { + aid = f2->file_actor_id(); } auto P = td::PromiseCreator::lambda( [aid, account_id, lt, promise = std::move(promise)](td::Result R) mutable { @@ -483,7 +483,7 @@ void ArchiveManager::get_block_by_lt(AccountIdPrefixFull account_id, LogicalTime td::actor::send_closure(aid, &ArchiveSlice::get_block_by_lt, account_id, lt, std::move(promise)); } }); - td::actor::send_closure(f->file_actor_id(), &ArchiveSlice::get_block_by_lt, account_id, lt, std::move(P)); + td::actor::send_closure(f1->file_actor_id(), &ArchiveSlice::get_block_by_lt, account_id, lt, std::move(P)); } else { promise.set_error(td::Status::Error(ErrorCode::notready, "lt not in db")); } @@ -558,7 +558,7 @@ void ArchiveManager::deleted_package(PackageId id, td::Promise promise auto it = m.find(id); CHECK(it != m.end()); CHECK(it->second.deleted); - it->second.clear_actor_id(); + it->second.file.reset(); promise.set_value(td::Unit()); } @@ -598,8 +598,8 @@ void ArchiveManager::load_package(PackageId id) { get_file_map(id).emplace(id, std::move(desc)); } -ArchiveManager::FileDescription *ArchiveManager::get_file_desc(ShardIdFull shard, PackageId id, BlockSeqno seqno, - UnixTime ts, LogicalTime lt, bool force) { +const ArchiveManager::FileDescription *ArchiveManager::get_file_desc(ShardIdFull shard, PackageId id, BlockSeqno seqno, + UnixTime ts, LogicalTime lt, bool force) { auto &f = get_file_map(id); auto it = f.find(id); if (it != f.end()) { @@ -607,7 +607,7 @@ ArchiveManager::FileDescription *ArchiveManager::get_file_desc(ShardIdFull shard return nullptr; } if (force && !id.temp) { - update_desc(it->second, shard, seqno, ts, lt); + update_desc(f, it->second, shard, seqno, ts, lt); } return &it->second; } @@ -618,17 +618,18 @@ ArchiveManager::FileDescription *ArchiveManager::get_file_desc(ShardIdFull shard return add_file_desc(shard, id, seqno, ts, lt); } -ArchiveManager::FileDescription *ArchiveManager::add_file_desc(ShardIdFull shard, PackageId id, BlockSeqno seqno, - UnixTime ts, LogicalTime lt) { +const ArchiveManager::FileDescription *ArchiveManager::add_file_desc(ShardIdFull shard, PackageId id, BlockSeqno seqno, + UnixTime ts, LogicalTime lt) { auto &f = get_file_map(id); CHECK(f.count(id) == 0); - FileDescription desc{id, false}; + FileDescription new_desc{id, false}; td::mkdir(db_root_ + id.path()).ensure(); std::string prefix = PSTRING() << db_root_ << id.path() << id.name(); - desc.file = td::actor::create_actor("slice", id.id, id.key, id.temp, false, db_root_); + new_desc.file = td::actor::create_actor("slice", id.id, id.key, id.temp, false, db_root_); + const FileDescription &desc = f.emplace(id, std::move(new_desc)); if (!id.temp) { - update_desc(desc, shard, seqno, ts, lt); + update_desc(f, desc, shard, seqno, ts, lt); } std::vector> vec; @@ -668,17 +669,16 @@ ArchiveManager::FileDescription *ArchiveManager::add_file_desc(ShardIdFull shard .ensure(); } index_->commit_transaction().ensure(); - - return &f.emplace(id, std::move(desc)).first->second; + return &desc; } -void ArchiveManager::update_desc(FileDescription &desc, ShardIdFull shard, BlockSeqno seqno, UnixTime ts, - LogicalTime lt) { +void ArchiveManager::update_desc(FileMap &f, const FileDescription &desc, ShardIdFull shard, BlockSeqno seqno, + UnixTime ts, LogicalTime lt) { auto it = desc.first_blocks.find(shard); if (it != desc.first_blocks.end() && it->second.seqno <= seqno) { return; } - desc.first_blocks[shard] = FileDescription::Desc{seqno, ts, lt}; + f.set_shard_first_block(desc, shard, FileDescription::Desc{seqno, ts, lt}); std::vector> vec; for (auto &e : desc.first_blocks) { vec.push_back(create_tl_object(e.first.workchain, e.first.shard, @@ -694,150 +694,91 @@ void ArchiveManager::update_desc(FileDescription &desc, ShardIdFull shard, Block index_->commit_transaction().ensure(); } -ArchiveManager::FileDescription *ArchiveManager::get_file_desc_by_seqno(ShardIdFull shard, BlockSeqno seqno, - bool key_block) { - auto &f = get_file_map(PackageId{0, key_block, false}); - for (auto it = f.rbegin(); it != f.rend(); it++) { - auto i = it->second.first_blocks.find(shard); - if (i != it->second.first_blocks.end() && i->second.seqno <= seqno) { - if (it->second.deleted) { - return nullptr; - } else { - return &it->second; - } - } - } - return nullptr; +const ArchiveManager::FileDescription *ArchiveManager::get_file_desc_by_seqno(ShardIdFull shard, BlockSeqno seqno, + bool key_block) { + return get_file_map(PackageId{0, key_block, false}).get_file_desc_by_seqno(shard, seqno); } -ArchiveManager::FileDescription *ArchiveManager::get_file_desc_by_unix_time(ShardIdFull shard, UnixTime ts, - bool key_block) { - auto &f = get_file_map(PackageId{0, key_block, false}); - for (auto it = f.rbegin(); it != f.rend(); it++) { - auto i = it->second.first_blocks.find(shard); - if (i != it->second.first_blocks.end() && i->second.ts <= ts) { - if (it->second.deleted) { - return nullptr; - } else { - return &it->second; - } - } - } - return nullptr; +const ArchiveManager::FileDescription *ArchiveManager::get_file_desc_by_unix_time(ShardIdFull shard, UnixTime ts, + bool key_block) { + return get_file_map(PackageId{0, key_block, false}).get_file_desc_by_unix_time(shard, ts); } -ArchiveManager::FileDescription *ArchiveManager::get_file_desc_by_lt(ShardIdFull shard, LogicalTime lt, - bool key_block) { - auto &f = get_file_map(PackageId{0, key_block, false}); - for (auto it = f.rbegin(); it != f.rend(); it++) { - auto i = it->second.first_blocks.find(shard); - if (i != it->second.first_blocks.end() && i->second.lt <= lt) { - if (it->second.deleted) { - return nullptr; - } else { - return &it->second; - } - } - } - return nullptr; +const ArchiveManager::FileDescription *ArchiveManager::get_file_desc_by_lt(ShardIdFull shard, LogicalTime lt, + bool key_block) { + return get_file_map(PackageId{0, key_block, false}).get_file_desc_by_lt(shard, lt); } -ArchiveManager::FileDescription *ArchiveManager::get_file_desc_by_seqno(AccountIdPrefixFull account, BlockSeqno seqno, - bool key_block) { - auto &f = get_file_map(PackageId{0, key_block, false}); +const ArchiveManager::FileDescription *ArchiveManager::get_file_desc_by_seqno(AccountIdPrefixFull account, + BlockSeqno seqno, bool key_block) { if (account.is_masterchain()) { return get_file_desc_by_seqno(ShardIdFull{masterchainId}, seqno, key_block); } - for (auto it = f.rbegin(); it != f.rend(); it++) { - if (it->second.deleted) { - continue; - } - bool found = false; - for (int i = 0; i < 60; i++) { - auto shard = shard_prefix(account, i); - auto it2 = it->second.first_blocks.find(shard); - if (it2 != it->second.first_blocks.end()) { - if (it2->second.seqno <= seqno) { - return &it->second; - } - found = true; - } else if (found) { - break; - } + auto &f = get_file_map(PackageId{0, key_block, false}); + const FileDescription *result = nullptr; + for (int i = 0; i <= 60; i++) { + const FileDescription *desc = f.get_file_desc_by_seqno(shard_prefix(account, i), seqno); + if (desc && (!result || result->id < desc->id)) { + result = desc; + } else if (result && (!desc || desc->id < result->id)) { + break; } } - return nullptr; + return result; } -ArchiveManager::FileDescription *ArchiveManager::get_file_desc_by_unix_time(AccountIdPrefixFull account, UnixTime ts, - bool key_block) { - auto &f = get_file_map(PackageId{0, key_block, false}); +const ArchiveManager::FileDescription *ArchiveManager::get_file_desc_by_unix_time(AccountIdPrefixFull account, + UnixTime ts, bool key_block) { if (account.is_masterchain()) { return get_file_desc_by_unix_time(ShardIdFull{masterchainId}, ts, key_block); } - for (auto it = f.rbegin(); it != f.rend(); it++) { - if (it->second.deleted) { - continue; - } - bool found = false; - for (int i = 0; i < 60; i++) { - auto shard = shard_prefix(account, i); - auto it2 = it->second.first_blocks.find(shard); - if (it2 != it->second.first_blocks.end()) { - if (it2->second.ts <= ts) { - return &it->second; - } - found = true; - } else if (found) { - break; - } + auto &f = get_file_map(PackageId{0, key_block, false}); + const FileDescription *result = nullptr; + for (int i = 0; i <= 60; i++) { + const FileDescription *desc = f.get_file_desc_by_unix_time(shard_prefix(account, i), ts); + if (desc && (!result || result->id < desc->id)) { + result = desc; + } else if (result && (!desc || desc->id < result->id)) { + break; } } - return nullptr; + return result; } -ArchiveManager::FileDescription *ArchiveManager::get_file_desc_by_lt(AccountIdPrefixFull account, LogicalTime lt, - bool key_block) { - auto &f = get_file_map(PackageId{0, key_block, false}); +const ArchiveManager::FileDescription *ArchiveManager::get_file_desc_by_lt(AccountIdPrefixFull account, LogicalTime lt, + bool key_block) { if (account.is_masterchain()) { return get_file_desc_by_lt(ShardIdFull{masterchainId}, lt, key_block); } - for (auto it = f.rbegin(); it != f.rend(); it++) { - if (it->second.deleted) { - continue; - } - bool found = false; - for (int i = 0; i < 60; i++) { - auto shard = shard_prefix(account, i); - auto it2 = it->second.first_blocks.find(shard); - if (it2 != it->second.first_blocks.end()) { - if (it2->second.lt <= lt) { - return &it->second; - } - found = true; - } else if (found) { - break; - } + auto &f = get_file_map(PackageId{0, key_block, false}); + const FileDescription *result = nullptr; + for (int i = 0; i <= 60; i++) { + const FileDescription *desc = f.get_file_desc_by_lt(shard_prefix(account, i), lt); + if (desc && (!result || result->id < desc->id)) { + result = desc; + } else if (result && (!desc || desc->id < result->id)) { + break; } } - return nullptr; + return result; } -ArchiveManager::FileDescription *ArchiveManager::get_next_file_desc(FileDescription *f) { - auto &m = get_file_map(f->id); - auto it = m.find(f->id); - CHECK(it != m.end()); - while (true) { - it++; - if (it == m.end()) { - return nullptr; - } else if (!it->second.deleted) { - return &it->second; +const ArchiveManager::FileDescription *ArchiveManager::get_next_file_desc(const FileDescription *f, + AccountIdPrefixFull shard, bool key_block) { + auto &m = get_file_map(PackageId{0, key_block, false}); + const FileDescription *result = nullptr; + for (int i = 0; i <= 60; i++) { + const FileDescription *desc = m.get_next_file_desc(shard_prefix(shard, i), f); + if (desc && (!result || desc->id < result->id)) { + result = desc; + } else if (result && (!desc || result->id < desc->id)) { + break; } } + return result; } -ArchiveManager::FileDescription *ArchiveManager::get_temp_file_desc_by_idx(PackageId idx) { +const ArchiveManager::FileDescription *ArchiveManager::get_temp_file_desc_by_idx(PackageId idx) { auto it = temp_files_.find(idx); if (it != temp_files_.end()) { if (it->second.deleted) { @@ -1257,14 +1198,100 @@ void ArchiveManager::truncate(BlockSeqno masterchain_seqno, ConstBlockHandle han } } -bool ArchiveManager::FileDescription::has_account_prefix(AccountIdPrefixFull account_id) const { - for (int i = 0; i < 60; i++) { - auto shard = shard_prefix(account_id, i); - if (first_blocks.count(shard)) { - return true; - } +void ArchiveManager::FileMap::shard_index_add(const FileDescription &desc) { + for (const auto &p : desc.first_blocks) { + ShardIndex &s = shards_[p.first]; + s.seqno_index_[p.second.seqno] = &desc; + s.lt_index_[p.second.lt] = &desc; + s.unix_time_index_[p.second.ts] = &desc; + s.packages_index_[desc.id] = &desc; + } +} + +void ArchiveManager::FileMap::shard_index_del(const FileDescription &desc) { + for (const auto &p : desc.first_blocks) { + ShardIndex &s = shards_[p.first]; + s.seqno_index_.erase(p.second.seqno); + s.lt_index_.erase(p.second.lt); + s.unix_time_index_.erase(p.second.ts); + s.packages_index_.erase(desc.id); + } +} + +void ArchiveManager::FileMap::set_shard_first_block(const FileDescription &desc, ShardIdFull shard, + FileDescription::Desc v) { + ShardIndex &s = shards_[shard]; + auto &d = const_cast(desc); + auto it = d.first_blocks.find(shard); + if (it != d.first_blocks.end()) { + s.seqno_index_.erase(it->second.seqno); + s.lt_index_.erase(it->second.lt); + s.unix_time_index_.erase(it->second.ts); + } + d.first_blocks[shard] = v; + s.seqno_index_[v.seqno] = &d; + s.lt_index_[v.lt] = &d; + s.unix_time_index_[v.ts] = &d; + s.packages_index_[d.id] = &d; +} + +const ArchiveManager::FileDescription *ArchiveManager::FileMap::get_file_desc_by_seqno(ShardIdFull shard, + BlockSeqno seqno) const { + auto it = shards_.find(shard); + if (it == shards_.end()) { + return nullptr; + } + const auto &map = it->second.seqno_index_; + auto it2 = map.upper_bound(seqno); + if (it2 == map.begin()) { + return nullptr; + } + --it2; + return it2->second->deleted ? nullptr : it2->second; +} + +const ArchiveManager::FileDescription *ArchiveManager::FileMap::get_file_desc_by_lt(ShardIdFull shard, + LogicalTime lt) const { + auto it = shards_.find(shard); + if (it == shards_.end()) { + return nullptr; + } + const auto &map = it->second.lt_index_; + auto it2 = map.upper_bound(lt); + if (it2 == map.begin()) { + return nullptr; + } + --it2; + return it2->second->deleted ? nullptr : it2->second; +} + +const ArchiveManager::FileDescription *ArchiveManager::FileMap::get_file_desc_by_unix_time(ShardIdFull shard, + UnixTime ts) const { + auto it = shards_.find(shard); + if (it == shards_.end()) { + return nullptr; + } + const auto &map = it->second.unix_time_index_; + auto it2 = map.upper_bound(ts); + if (it2 == map.begin()) { + return nullptr; + } + --it2; + return it2->second->deleted ? nullptr : it2->second; +} + +const ArchiveManager::FileDescription *ArchiveManager::FileMap::get_next_file_desc(ShardIdFull shard, + const FileDescription *desc) const { + auto it = shards_.find(shard); + if (it == shards_.end()) { + return nullptr; + } + const auto &map = it->second.packages_index_; + auto it2 = desc ? map.upper_bound(desc->id) : map.begin(); + if (it2 == map.end()) { + return nullptr; } - return false; + return it2->second->deleted ? nullptr : it2->second; } } // namespace validator diff --git a/validator/db/archive-manager.hpp b/validator/db/archive-manager.hpp index 79e6a2d78..e50087642 100644 --- a/validator/db/archive-manager.hpp +++ b/validator/db/archive-manager.hpp @@ -71,7 +71,6 @@ class ArchiveManager : public td::actor::Actor { void start_up() override; - void begin_transaction(); void commit_transaction(); void set_async_mode(bool mode, td::Promise promise); @@ -94,26 +93,83 @@ class ArchiveManager : public td::actor::Actor { auto file_actor_id() const { return file.get(); } - void clear_actor_id() { - file.reset(); - } - bool has_account_prefix(AccountIdPrefixFull account_id) const; PackageId id; - bool deleted; + mutable bool deleted; std::map first_blocks; - td::actor::ActorOwn file; + mutable td::actor::ActorOwn file; }; - std::map files_; - std::map key_files_; - std::map temp_files_; + class FileMap { + public: + std::map::const_iterator begin() const { + return files_.cbegin(); + } + std::map::const_iterator end() const { + return files_.cend(); + } + std::map::const_reverse_iterator rbegin() const { + return files_.crbegin(); + } + std::map::const_reverse_iterator rend() const { + return files_.crend(); + } + std::map::const_iterator find(PackageId x) const { + return files_.find(x); + } + size_t count(const PackageId &x) const { + return files_.count(x); + } + size_t size() const { + return files_.size(); + } + std::map::const_iterator lower_bound(const PackageId &x) const { + return files_.lower_bound(x); + } + std::map::const_iterator upper_bound(const PackageId &x) const { + return files_.upper_bound(x); + } + void clear() { + files_.clear(); + shards_.clear(); + } + const FileDescription &emplace(const PackageId &id, FileDescription desc) { + auto it = files_.emplace(id, std::move(desc)); + if (it.second) { + shard_index_add(it.first->second); + } + return it.first->second; + } + void erase(std::map::const_iterator it) { + shard_index_del(it->second); + files_.erase(it); + } + void set_shard_first_block(const FileDescription &desc, ShardIdFull shard, FileDescription::Desc v); + const FileDescription *get_file_desc_by_seqno(ShardIdFull shard, BlockSeqno seqno) const; + const FileDescription *get_file_desc_by_lt(ShardIdFull shard, LogicalTime lt) const; + const FileDescription *get_file_desc_by_unix_time(ShardIdFull shard, UnixTime ts) const; + const FileDescription *get_next_file_desc(ShardIdFull shard, const FileDescription *desc) const; + + private: + std::map files_; + struct ShardIndex { + std::map seqno_index_; + std::map lt_index_; + std::map unix_time_index_; + std::map packages_index_; + }; + std::map shards_; + + void shard_index_add(const FileDescription &desc); + void shard_index_del(const FileDescription &desc); + }; + FileMap files_, key_files_, temp_files_; BlockSeqno finalized_up_to_{0}; bool async_mode_ = false; bool huge_transaction_started_ = false; td::uint32 huge_transaction_size_ = 0; - auto &get_file_map(const PackageId &p) { + FileMap &get_file_map(const PackageId &p) { return p.key ? key_files_ : p.temp ? temp_files_ : files_; } @@ -126,18 +182,19 @@ class ArchiveManager : public td::actor::Actor { void get_handle_finish(BlockHandle handle, td::Promise promise); void get_file_short_cont(FileReference ref_id, PackageId idx, td::Promise promise); - FileDescription *get_file_desc(ShardIdFull shard, PackageId id, BlockSeqno seqno, UnixTime ts, LogicalTime lt, - bool force); - FileDescription *add_file_desc(ShardIdFull shard, PackageId id, BlockSeqno seqno, UnixTime ts, LogicalTime lt); - void update_desc(FileDescription &desc, ShardIdFull shard, BlockSeqno seqno, UnixTime ts, LogicalTime lt); - FileDescription *get_file_desc_by_seqno(ShardIdFull shard, BlockSeqno seqno, bool key_block); - FileDescription *get_file_desc_by_lt(ShardIdFull shard, LogicalTime lt, bool key_block); - FileDescription *get_file_desc_by_unix_time(ShardIdFull shard, UnixTime ts, bool key_block); - FileDescription *get_file_desc_by_seqno(AccountIdPrefixFull shard, BlockSeqno seqno, bool key_block); - FileDescription *get_file_desc_by_lt(AccountIdPrefixFull shard, LogicalTime lt, bool key_block); - FileDescription *get_file_desc_by_unix_time(AccountIdPrefixFull shard, UnixTime ts, bool key_block); - FileDescription *get_next_file_desc(FileDescription *f); - FileDescription *get_temp_file_desc_by_idx(PackageId idx); + const FileDescription *get_file_desc(ShardIdFull shard, PackageId id, BlockSeqno seqno, UnixTime ts, LogicalTime lt, + bool force); + const FileDescription *add_file_desc(ShardIdFull shard, PackageId id, BlockSeqno seqno, UnixTime ts, LogicalTime lt); + void update_desc(FileMap &f, const FileDescription &desc, ShardIdFull shard, BlockSeqno seqno, UnixTime ts, + LogicalTime lt); + const FileDescription *get_file_desc_by_seqno(ShardIdFull shard, BlockSeqno seqno, bool key_block); + const FileDescription *get_file_desc_by_lt(ShardIdFull shard, LogicalTime lt, bool key_block); + const FileDescription *get_file_desc_by_unix_time(ShardIdFull shard, UnixTime ts, bool key_block); + const FileDescription *get_file_desc_by_seqno(AccountIdPrefixFull shard, BlockSeqno seqno, bool key_block); + const FileDescription *get_file_desc_by_lt(AccountIdPrefixFull shard, LogicalTime lt, bool key_block); + const FileDescription *get_file_desc_by_unix_time(AccountIdPrefixFull shard, UnixTime ts, bool key_block); + const FileDescription *get_next_file_desc(const FileDescription *f, AccountIdPrefixFull shard, bool key_block); + const FileDescription *get_temp_file_desc_by_idx(PackageId idx); PackageId get_max_temp_file_desc_idx(); PackageId get_prev_temp_file_desc_idx(PackageId id); From 444dda8d6264bc00ab7c63ea8560ba336210b47b Mon Sep 17 00:00:00 2001 From: EmelyanenkoK Date: Fri, 28 Apr 2023 16:42:11 +0300 Subject: [PATCH 10/20] Revert too strictening of queue_cleanup_timeout --- validator/impl/collator.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/validator/impl/collator.cpp b/validator/impl/collator.cpp index 1c3cafe6b..4e68b07ca 100644 --- a/validator/impl/collator.cpp +++ b/validator/impl/collator.cpp @@ -68,7 +68,7 @@ Collator::Collator(ShardIdFull shard, bool is_hardfork, UnixTime min_ts, BlockId , manager(manager) , timeout(timeout) // default timeout is 10 seconds, declared in validator/validator-group.cpp:generate_block_candidate:run_collate_query - , queue_cleanup_timeout_(td::Timestamp::at(timeout.at() - 9.0)) + , queue_cleanup_timeout_(td::Timestamp::at(timeout.at() - 5.0)) , soft_timeout_(td::Timestamp::at(timeout.at() - 3.0)) , medium_timeout_(td::Timestamp::at(timeout.at() - 1.5)) , main_promise(std::move(promise)) From 87d4f4bc7fe63a92b22dcc0cc1a5e10f9e01d8b1 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Tue, 2 May 2023 18:32:20 +0300 Subject: [PATCH 11/20] Fix saving list of packages in archive manager --- validator/db/archive-manager.cpp | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/validator/db/archive-manager.cpp b/validator/db/archive-manager.cpp index 7c2aefe3e..91b7b880b 100644 --- a/validator/db/archive-manager.cpp +++ b/validator/db/archive-manager.cpp @@ -563,6 +563,11 @@ void ArchiveManager::deleted_package(PackageId id, td::Promise promise } void ArchiveManager::load_package(PackageId id) { + auto &m = get_file_map(id); + if (m.count(id)) { + LOG(WARNING) << "Duplicate id " << id.name(); + return; + } auto key = create_serialize_tl_object(id.id, id.key, id.temp); std::string value; @@ -595,7 +600,7 @@ void ArchiveManager::load_package(PackageId id) { desc.file = td::actor::create_actor("slice", id.id, id.key, id.temp, false, db_root_); - get_file_map(id).emplace(id, std::move(desc)); + m.emplace(id, std::move(desc)); } const ArchiveManager::FileDescription *ArchiveManager::get_file_desc(ShardIdFull shard, PackageId id, BlockSeqno seqno, @@ -653,7 +658,6 @@ const ArchiveManager::FileDescription *ArchiveManager::add_file_desc(ShardIdFull for (auto &e : temp_files_) { tt.push_back(e.first.id); } - (id.temp ? tt : (id.key ? tk : t)).push_back(id.id); index_ ->set(create_serialize_tl_object().as_slice(), create_serialize_tl_object(std::move(t), std::move(tk), std::move(tt)) From a78adf3062b73136a4f8cf558b3b46f92498af30 Mon Sep 17 00:00:00 2001 From: EmelyanenkoK Date: Wed, 3 May 2023 14:18:18 +0300 Subject: [PATCH 12/20] Increase limit for t_Block.validate, simplify validating StateInit (#691) Co-authored-by: SpyCheese --- crypto/block/block-parse.cpp | 32 +++++++++++++++++++++++++++++++ crypto/block/block-parse.h | 4 ++++ crypto/block/block.tlb | 9 +++++++-- crypto/block/transaction.cpp | 9 +++++++-- validator/impl/collator.cpp | 11 +++++++++-- validator/impl/validate-query.cpp | 8 ++++---- 6 files changed, 63 insertions(+), 10 deletions(-) diff --git a/crypto/block/block-parse.cpp b/crypto/block/block-parse.cpp index e9eb8209a..7d51b2e23 100644 --- a/crypto/block/block-parse.cpp +++ b/crypto/block/block-parse.cpp @@ -2292,5 +2292,37 @@ bool Aug_ShardFees::eval_leaf(vm::CellBuilder& cb, vm::CellSlice& cs) const { const Aug_ShardFees aug_ShardFees; +bool validate_message_libs(const td::Ref &cell) { + gen::Message::Record rec; + if (!type_unpack_cell(cell, gen::t_Message_Any, rec)) { + return false; + } + vm::CellSlice& state_init = rec.init.write(); + if (!state_init.fetch_long(1)) { + return true; + } + if (state_init.fetch_long(1)) { + return gen::t_StateInitWithLibs.validate_ref(state_init.prefetch_ref()); + } else { + return gen::t_StateInitWithLibs.validate_csr(rec.init); + } +} + +bool validate_message_relaxed_libs(const td::Ref &cell) { + gen::MessageRelaxed::Record rec; + if (!type_unpack_cell(cell, gen::t_MessageRelaxed_Any, rec)) { + return false; + } + vm::CellSlice& state_init = rec.init.write(); + if (!state_init.fetch_long(1)) { + return true; + } + if (state_init.fetch_long(1)) { + return gen::t_StateInitWithLibs.validate_ref(state_init.prefetch_ref()); + } else { + return gen::t_StateInitWithLibs.validate_csr(rec.init); + } +} + } // namespace tlb } // namespace block diff --git a/crypto/block/block-parse.h b/crypto/block/block-parse.h index ad4faec08..c0b117452 100644 --- a/crypto/block/block-parse.h +++ b/crypto/block/block-parse.h @@ -1113,5 +1113,9 @@ struct Aug_ShardFees final : AugmentationCheckData { extern const Aug_ShardFees aug_ShardFees; +// Validate dict of libraries in message: used when sending and receiving message +bool validate_message_libs(const td::Ref &cell); +bool validate_message_relaxed_libs(const td::Ref &cell); + } // namespace tlb } // namespace block diff --git a/crypto/block/block.tlb b/crypto/block/block.tlb index 0c8fdb83e..6ab035089 100644 --- a/crypto/block/block.tlb +++ b/crypto/block/block.tlb @@ -143,8 +143,13 @@ tick_tock$_ tick:Bool tock:Bool = TickTock; _ split_depth:(Maybe (## 5)) special:(Maybe TickTock) code:(Maybe ^Cell) data:(Maybe ^Cell) - library:(HashmapE 256 SimpleLib) = StateInit; - + library:(Maybe ^Cell) = StateInit; + +// StateInitWithLibs is used to validate sent and received messages +_ split_depth:(Maybe (## 5)) special:(Maybe TickTock) + code:(Maybe ^Cell) data:(Maybe ^Cell) + library:(HashmapE 256 SimpleLib) = StateInitWithLibs; + simple_lib$_ public:Bool root:^Cell = SimpleLib; message$_ {X:Type} info:CommonMsgInfo diff --git a/crypto/block/transaction.cpp b/crypto/block/transaction.cpp index fcabf0c09..1a8f111c6 100644 --- a/crypto/block/transaction.cpp +++ b/crypto/block/transaction.cpp @@ -635,14 +635,15 @@ bool Transaction::unpack_input_msg(bool ihr_delivered, const ActionPhaseConfig* vm::CellBuilder cb; if (!(cs.advance(2) && block::gen::t_StateInit.fetch_to(cs, state_init) && cb.append_cellslice_bool(std::move(state_init)) && cb.finalize_to(in_msg_state) && - block::gen::t_StateInit.validate_ref(in_msg_state))) { + block::gen::t_StateInitWithLibs.validate_ref(in_msg_state))) { LOG(DEBUG) << "cannot parse StateInit in inbound message"; return false; } break; } case 3: { // (just$1 (right$1 _:^StateInit )) - if (!(cs.advance(2) && cs.fetch_ref_to(in_msg_state) && block::gen::t_StateInit.validate_ref(in_msg_state))) { + if (!(cs.advance(2) && cs.fetch_ref_to(in_msg_state) && + block::gen::t_StateInitWithLibs.validate_ref(in_msg_state))) { LOG(DEBUG) << "cannot parse ^StateInit in inbound message"; return false; } @@ -1534,6 +1535,10 @@ int Transaction::try_action_send_msg(const vm::CellSlice& cs0, ActionPhase& ap, if (!tlb::type_unpack_cell(act_rec.out_msg, block::gen::t_MessageRelaxed_Any, msg)) { return -1; } + if (!block::tlb::validate_message_relaxed_libs(act_rec.out_msg)) { + LOG(DEBUG) << "outbound message has invalid libs in StateInit"; + return -1; + } if (redoing >= 1) { if (msg.init->size_refs() >= 2) { LOG(DEBUG) << "moving the StateInit of a suggested outbound message into a separate cell"; diff --git a/validator/impl/collator.cpp b/validator/impl/collator.cpp index 4e68b07ca..9a6c512b5 100644 --- a/validator/impl/collator.cpp +++ b/validator/impl/collator.cpp @@ -2565,7 +2565,7 @@ bool Collator::process_inbound_message(Ref enq_msg, ton::LogicalT return false; } if (!block::tlb::t_MsgEnvelope.validate_ref(msg_env)) { - LOG(ERROR) << "inbound internal MsgEnvelope is invalid according to automated checks"; + LOG(ERROR) << "inbound internal MsgEnvelope is invalid according to hand-written checks"; return false; } // 1. unpack MsgEnvelope @@ -2590,6 +2590,10 @@ bool Collator::process_inbound_message(Ref enq_msg, ton::LogicalT "its contents"; return false; } + if (!block::tlb::validate_message_libs(env.msg)) { + LOG(ERROR) << "inbound internal message has invalid StateInit"; + return false; + } // 2.0. update last_proc_int_msg if (!update_last_proc_int_msg(std::pair(lt, env.msg->get_hash().bits()))) { return fatal_error("processing a message AFTER a newer message has been processed"); @@ -3876,7 +3880,7 @@ bool Collator::create_block() { } if (verify >= 1) { LOG(INFO) << "verifying new Block"; - if (!block::gen::t_Block.validate_ref(1000000, new_block)) { + if (!block::gen::t_Block.validate_ref(10000000, new_block)) { return fatal_error("new Block failed to pass automatic validity tests"); } } @@ -4030,6 +4034,9 @@ td::Result Collator::register_external_message_cell(Ref ext_msg, if (!block::tlb::t_Message.validate_ref(256, ext_msg)) { return td::Status::Error("external message is not a (Message Any) according to hand-written checks"); } + if (!block::tlb::validate_message_libs(ext_msg)) { + return td::Status::Error("external message has invalid libs in StateInit"); + } block::gen::CommonMsgInfo::Record_ext_in_msg_info info; if (!tlb::unpack_cell_inexact(ext_msg, info)) { return td::Status::Error("cannot unpack external message header"); diff --git a/validator/impl/validate-query.cpp b/validator/impl/validate-query.cpp index 0e1b11998..b47254c34 100644 --- a/validator/impl/validate-query.cpp +++ b/validator/impl/validate-query.cpp @@ -2111,13 +2111,13 @@ bool ValidateQuery::unpack_block_data() { auto outmsg_cs = vm::load_cell_slice_ref(std::move(extra.out_msg_descr)); // run some hand-written checks from block::tlb:: // (automatic tests from block::gen:: have been already run for the entire block) - if (!block::tlb::t_InMsgDescr.validate_upto(1000000, *inmsg_cs)) { + if (!block::tlb::t_InMsgDescr.validate_upto(10000000, *inmsg_cs)) { return reject_query("InMsgDescr of the new block failed to pass handwritten validity tests"); } - if (!block::tlb::t_OutMsgDescr.validate_upto(1000000, *outmsg_cs)) { + if (!block::tlb::t_OutMsgDescr.validate_upto(10000000, *outmsg_cs)) { return reject_query("OutMsgDescr of the new block failed to pass handwritten validity tests"); } - if (!block::tlb::t_ShardAccountBlocks.validate_ref(1000000, extra.account_blocks)) { + if (!block::tlb::t_ShardAccountBlocks.validate_ref(10000000, extra.account_blocks)) { return reject_query("ShardAccountBlocks of the new block failed to pass handwritten validity tests"); } in_msg_dict_ = std::make_unique(std::move(inmsg_cs), 256, block::tlb::aug_InMsgDescr); @@ -5507,7 +5507,7 @@ bool ValidateQuery::try_validate() { } } LOG(INFO) << "running automated validity checks for block candidate " << id_.to_str(); - if (!block::gen::t_Block.validate_ref(1000000, block_root_)) { + if (!block::gen::t_Block.validate_ref(10000000, block_root_)) { return reject_query("block "s + id_.to_str() + " failed to pass automated validity checks"); } if (!fix_all_processed_upto()) { From 1696ebfa203f2c93c9956e8bac1ca89f7d3446b6 Mon Sep 17 00:00:00 2001 From: EmelyanenkoK Date: Thu, 4 May 2023 14:45:42 +0300 Subject: [PATCH 13/20] Account for unprocessed messages in estimate_block_size; check consensus_config limits in collator (#692) Co-authored-by: SpyCheese --- crypto/block/block.cpp | 2 +- crypto/block/block.h | 3 ++- validator/impl/collator.cpp | 14 ++++++++++++++ 3 files changed, 17 insertions(+), 2 deletions(-) diff --git a/crypto/block/block.cpp b/crypto/block/block.cpp index 9e2caef5e..e1790cbfe 100644 --- a/crypto/block/block.cpp +++ b/crypto/block/block.cpp @@ -715,7 +715,7 @@ td::uint64 BlockLimitStatus::estimate_block_size(const vm::NewCellStorageStat::S sum += *extra; } return 2000 + (sum.bits >> 3) + sum.cells * 12 + sum.internal_refs * 3 + sum.external_refs * 40 + accounts * 200 + - transactions * 200 + (extra ? 200 : 0); + transactions * 200 + (extra ? 200 : 0) + extra_out_msgs * 300; } int BlockLimitStatus::classify() const { diff --git a/crypto/block/block.h b/crypto/block/block.h index 6c460e318..f5b47a63a 100644 --- a/crypto/block/block.h +++ b/crypto/block/block.h @@ -261,7 +261,7 @@ struct BlockLimitStatus { ton::LogicalTime cur_lt; td::uint64 gas_used{}; vm::NewCellStorageStat st_stat; - unsigned accounts{}, transactions{}; + unsigned accounts{}, transactions{}, extra_out_msgs{}; BlockLimitStatus(const BlockLimits& limits_, ton::LogicalTime lt = 0) : limits(limits_), cur_lt(std::max(limits_.start_lt, lt)) { } @@ -270,6 +270,7 @@ struct BlockLimitStatus { st_stat.set_zero(); transactions = accounts = 0; gas_used = 0; + extra_out_msgs = 0; } td::uint64 estimate_block_size(const vm::NewCellStorageStat::Stat* extra = nullptr) const; int classify() const; diff --git a/validator/impl/collator.cpp b/validator/impl/collator.cpp index 9a6c512b5..34dfca617 100644 --- a/validator/impl/collator.cpp +++ b/validator/impl/collator.cpp @@ -2961,6 +2961,7 @@ bool Collator::process_new_messages(bool enqueue_only) { while (!new_msgs.empty()) { block::NewOutMsg msg = new_msgs.top(); new_msgs.pop(); + block_limit_status_->extra_out_msgs--; if (block_full_ && !enqueue_only) { LOG(INFO) << "BLOCK FULL, enqueue all remaining new messages"; enqueue_only = true; @@ -2982,6 +2983,7 @@ void Collator::register_new_msg(block::NewOutMsg new_msg) { min_new_msg_lt = new_msg.lt; } new_msgs.push(std::move(new_msg)); + block_limit_status_->extra_out_msgs++; } void Collator::register_new_msgs(block::transaction::Transaction& trans) { @@ -3972,6 +3974,18 @@ bool Collator::create_block_candidate() { ton::BlockIdExt{ton::BlockId{shard_, new_block_seqno}, new_block->get_hash().bits(), block::compute_file_hash(blk_slice.as_slice())}, block::compute_file_hash(cdata_slice.as_slice()), blk_slice.clone(), cdata_slice.clone()); + // 3.1 check block and collated data size + auto consensus_config = config_->get_consensus_config(); + if (block_candidate->data.size() > consensus_config.max_block_size) { + return fatal_error(PSTRING() << "block size (" << block_candidate->data.size() + << ") exceeds the limit in consensus config (" << consensus_config.max_block_size + << ")"); + } + if (block_candidate->collated_data.size() > consensus_config.max_collated_data_size) { + return fatal_error(PSTRING() << "collated data size (" << block_candidate->collated_data.size() + << ") exceeds the limit in consensus config (" + << consensus_config.max_collated_data_size << ")"); + } // 4. save block candidate LOG(INFO) << "saving new BlockCandidate"; td::actor::send_closure_later(manager, &ValidatorManager::set_block_candidate, block_candidate->id, From f8b585df39630b63cdc0de44056621eb65372291 Mon Sep 17 00:00:00 2001 From: neodiX42 Date: Thu, 4 May 2023 08:49:43 -0500 Subject: [PATCH 14/20] Fix missing git revision when built with Nix. (#680) --- flake.nix | 3 +++ git_watcher.cmake | 4 ++++ 2 files changed, 7 insertions(+) diff --git a/flake.nix b/flake.nix index e7b0422e0..d3fca85c3 100644 --- a/flake.nix +++ b/flake.nix @@ -50,6 +50,9 @@ "-static-libstdc++" ]); + GIT_REVISION = if self ? rev then self.rev else "dirty"; + GIT_REVISION_DATE = (builtins.concatStringsSep "-" (builtins.match "(.{4})(.{2})(.{2}).*" self.lastModifiedDate)) + " " + (builtins.concatStringsSep ":" (builtins.match "^........(.{2})(.{2})(.{2}).*" self.lastModifiedDate)); + postInstall = '' moveToOutput bin "$bin" ''; diff --git a/git_watcher.cmake b/git_watcher.cmake index ab135e808..78e57ba14 100644 --- a/git_watcher.cmake +++ b/git_watcher.cmake @@ -139,6 +139,8 @@ function(GetGitState _working_dir) RunGitCommand(show -s "--format=%H" ${object}) if(exit_code EQUAL 0) set(ENV{GIT_HEAD_SHA1} ${output}) + else() + set(ENV{GIT_HEAD_SHA1} "$ENV{GIT_REVISION}") endif() RunGitCommand(show -s "--format=%an" ${object}) @@ -154,6 +156,8 @@ function(GetGitState _working_dir) RunGitCommand(show -s "--format=%ci" ${object}) if(exit_code EQUAL 0) set(ENV{GIT_COMMIT_DATE_ISO8601} "${output}") + else() + set(ENV{GIT_COMMIT_DATE_ISO8601} "$ENV{GIT_REVISION_DATE}") endif() RunGitCommand(show -s "--format=%s" ${object}) From 1aadc80742b55487355412633d2241bbf64747bf Mon Sep 17 00:00:00 2001 From: neodiX42 Date: Thu, 4 May 2023 08:50:25 -0500 Subject: [PATCH 15/20] More portable (#681) * integrate static openssl into linux binaries; do not use AVX and AVX2 CPU instructions in windows binaries. * integrate static openssl into linux binaries; do not use AVX and AVX2 CPU instructions in windows binaries. --- .github/workflows/ubuntu-22.04-compile.yml | 12 +++++++++++- .github/workflows/ubuntu-compile.yml | 13 ++++++++++++- .github/workflows/win-2019-compile.yml | 7 +------ 3 files changed, 24 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ubuntu-22.04-compile.yml b/.github/workflows/ubuntu-22.04-compile.yml index d27d1b4da..8032e498a 100644 --- a/.github/workflows/ubuntu-22.04-compile.yml +++ b/.github/workflows/ubuntu-22.04-compile.yml @@ -27,9 +27,19 @@ jobs: export CC=$(which clang) export CXX=$(which clang++) export CCACHE_DISABLE=1 + + git clone https://github.com/openssl/openssl openssl_1_1_1 + cd openssl_1_1_1 + git checkout OpenSSL_1_1_1-stable + ./config + make build_libs -j4 + + cd .. + rootPath=`pwd` mkdir build cd build - cmake -GNinja -DCMAKE_BUILD_TYPE=Release -DPORTABLE=1 -DTON_ARCH= -DCMAKE_CXX_FLAGS="-mavx2" .. + + cmake -GNinja -DOPENSSL_FOUND=1 -DOPENSSL_INCLUDE_DIR=$rootPath/openssl_1_1_1/include -DOPENSSL_CRYPTO_LIBRARY=$rootPath/openssl_1_1_1/libcrypto.a -DCMAKE_BUILD_TYPE=Release -DPORTABLE=1 -DTON_ARCH= -DCMAKE_CXX_FLAGS="-mavx2" .. ninja storage-daemon storage-daemon-cli fift func tonlib tonlibjson tonlib-cli validator-engine lite-client pow-miner validator-engine-console generate-random-id json2tlo dht-server http-proxy rldp-http-proxy adnl-proxy create-state - name: Find & copy binaries diff --git a/.github/workflows/ubuntu-compile.yml b/.github/workflows/ubuntu-compile.yml index dd246a39e..b303e143b 100644 --- a/.github/workflows/ubuntu-compile.yml +++ b/.github/workflows/ubuntu-compile.yml @@ -30,9 +30,20 @@ jobs: export CC=$(which clang) export CXX=$(which clang++) export CCACHE_DISABLE=1 + mkdir build-${{ matrix.os }} cd build-${{ matrix.os }} - cmake -GNinja -DCMAKE_BUILD_TYPE=Release -DPORTABLE=1 -DTON_ARCH= -DCMAKE_CXX_FLAGS="-mavx2" .. + + git clone https://github.com/openssl/openssl openssl_1_1_1 + cd openssl_1_1_1 + git checkout OpenSSL_1_1_1-stable + ./config + make build_libs -j4 + + cd .. + buildPath=`pwd` + + cmake -GNinja -DOPENSSL_FOUND=1 -DOPENSSL_INCLUDE_DIR=$buildPath/openssl_1_1_1/include -DOPENSSL_CRYPTO_LIBRARY=$buildPath/openssl_1_1_1/libcrypto.a -DCMAKE_BUILD_TYPE=Release -DPORTABLE=1 -DTON_ARCH= -DCMAKE_CXX_FLAGS="-mavx2" .. ninja storage-daemon storage-daemon-cli fift func tonlib tonlibjson tonlib-cli validator-engine lite-client pow-miner validator-engine-console generate-random-id json2tlo dht-server http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork - name: Find & copy binaries diff --git a/.github/workflows/win-2019-compile.yml b/.github/workflows/win-2019-compile.yml index bd174a15c..b0c999714 100644 --- a/.github/workflows/win-2019-compile.yml +++ b/.github/workflows/win-2019-compile.yml @@ -48,18 +48,13 @@ jobs: curl -Lo libmicrohttpd-latest-w32-bin.zip https://ftpmirror.gnu.org/libmicrohttpd/libmicrohttpd-latest-w32-bin.zip unzip libmicrohttpd-latest-w32-bin.zip - - name: Install pre-compiled Readline Win64 - run: | - curl -Lo readline-5.0-1-lib.zip https://github.com/neodiX42/precompiled-openssl-win64/raw/main/readline-5.0-1-lib.zip - unzip readline-5.0-1-lib.zip - - name: Compile run: | set root=%cd% echo %root% mkdir build cd build - cmake -DREADLINE_INCLUDE_DIR=%root%\readline-5.0-1-lib\include\readline -DREADLINE_LIBRARY=%root%\readline-5.0-1-lib\lib\readline.lib -DZLIB_FOUND=1 -DMHD_FOUND=1 -DMHD_LIBRARY=%root%\libmicrohttpd-0.9.75-w32-bin\x86_64\VS2019\Release-static\libmicrohttpd.lib -DMHD_INCLUDE_DIR=%root%\libmicrohttpd-0.9.75-w32-bin\x86_64\VS2019\Release-static -DZLIB_INCLUDE_DIR=%root%\zlib -DZLIB_LIBRARY=%root%\zlib\contrib\vstudio\vc14\x64\ZlibStatReleaseWithoutAsm\zlibstat.lib -DOPENSSL_FOUND=1 -DOPENSSL_INCLUDE_DIR=%root%/openssl-1.1/x64/include -DOPENSSL_CRYPTO_LIBRARY=%root%/openssl-1.1/x64/lib/libcrypto.lib -DCMAKE_CXX_FLAGS="/DTD_WINDOWS=1 /EHsc /bigobj /W0" .. + cmake -DPORTABLE=1 -DZLIB_FOUND=1 -DMHD_FOUND=1 -DMHD_LIBRARY=%root%\libmicrohttpd-0.9.75-w32-bin\x86_64\VS2019\Release-static\libmicrohttpd.lib -DMHD_INCLUDE_DIR=%root%\libmicrohttpd-0.9.75-w32-bin\x86_64\VS2019\Release-static -DZLIB_INCLUDE_DIR=%root%\zlib -DZLIB_LIBRARY=%root%\zlib\contrib\vstudio\vc14\x64\ZlibStatReleaseWithoutAsm\zlibstat.lib -DOPENSSL_FOUND=1 -DOPENSSL_INCLUDE_DIR=%root%/openssl-1.1/x64/include -DOPENSSL_CRYPTO_LIBRARY=%root%/openssl-1.1/x64/lib/libcrypto.lib -DCMAKE_CXX_FLAGS="/DTD_WINDOWS=1 /EHsc /bigobj /W0" .. cmake --build . --target storage-daemon storage-daemon-cli blockchain-explorer fift func tonlib tonlibjson tonlib-cli validator-engine lite-client pow-miner validator-engine-console generate-random-id json2tlo dht-server http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork --config Release - name: Show executables From 7878578dba955d51ea4f0e0753070f92a9c0f636 Mon Sep 17 00:00:00 2001 From: neodiX42 Date: Fri, 5 May 2023 03:50:46 -0500 Subject: [PATCH 16/20] Add day to the git tag name (#652) * Add day to the git tag name * Change release tag generation method. v2023.04 v2023.04-1 v2023.04-2 v2023.04-3 etc * add git fetch to get all tags * add git fetch to get all tags --- .github/workflows/create-release.yml | 181 ++++++++++++++------------- recent_changelog.md | 6 + 2 files changed, 101 insertions(+), 86 deletions(-) create mode 100644 recent_changelog.md diff --git a/.github/workflows/create-release.yml b/.github/workflows/create-release.yml index 85d0dc259..cb6651220 100644 --- a/.github/workflows/create-release.yml +++ b/.github/workflows/create-release.yml @@ -27,21 +27,21 @@ jobs: workflow_conclusion: success skip_unpack: false -# - name: Download Linux arm64 artifacts -# uses: dawidd6/action-download-artifact@v2 -# with: -# workflow: ton-aarch64-linux.yml -# path: artifacts -# workflow_conclusion: success -# skip_unpack: true -# -# - name: Download and unzip Linux arm64 artifacts -# uses: dawidd6/action-download-artifact@v2 -# with: -# workflow: ton-aarch64-linux.yml -# path: artifacts -# workflow_conclusion: success -# skip_unpack: false + # - name: Download Linux arm64 artifacts + # uses: dawidd6/action-download-artifact@v2 + # with: + # workflow: ton-aarch64-linux.yml + # path: artifacts + # workflow_conclusion: success + # skip_unpack: true + # + # - name: Download and unzip Linux arm64 artifacts + # uses: dawidd6/action-download-artifact@v2 + # with: + # workflow: ton-aarch64-linux.yml + # path: artifacts + # workflow_conclusion: success + # skip_unpack: false - name: Download Mac x86-64 artifacts uses: dawidd6/action-download-artifact@v2 @@ -59,21 +59,21 @@ jobs: workflow_conclusion: success skip_unpack: false -# - name: Download Mac arm64 artifacts -# uses: dawidd6/action-download-artifact@v2 -# with: -# workflow: ton-aarch64-macos.yml -# path: artifacts -# workflow_conclusion: success -# skip_unpack: true -# -# - name: Download and unzip Mac arm64 artifacts -# uses: dawidd6/action-download-artifact@v2 -# with: -# workflow: ton-aarch64-macos.yml -# path: artifacts -# workflow_conclusion: success -# skip_unpack: false + # - name: Download Mac arm64 artifacts + # uses: dawidd6/action-download-artifact@v2 + # with: + # workflow: ton-aarch64-macos.yml + # path: artifacts + # workflow_conclusion: success + # skip_unpack: true + # + # - name: Download and unzip Mac arm64 artifacts + # uses: dawidd6/action-download-artifact@v2 + # with: + # workflow: ton-aarch64-macos.yml + # path: artifacts + # workflow_conclusion: success + # skip_unpack: false - name: Download Windows artifacts uses: dawidd6/action-download-artifact@v2 @@ -104,26 +104,35 @@ jobs: tree artifacts -# create release + # create release - name: Read Changelog.md and use it as a body of new release id: read_release shell: bash run: | - r=$(cat Changelog.md) + r=$(cat recent_changelog.md) r="${r//'%'/'%25'}" r="${r//$'\n'/'%0A'}" r="${r//$'\r'/'%0D'}" echo "::set-output name=CHANGELOG_BODY::$r" - - name: Get current date - id: date - run: echo "::set-output name=date::$(date +'%Y.%m')" + - name: Get next tag + id: tag + run: | + git fetch --all --tags + git tag -l + NEW_TAG=v$(date +'%Y.%m') + FOUND=$(git tag -l | grep $NEW_TAG | wc -l) + if [ $FOUND -eq 0 ]; then + echo "TAG=$NEW_TAG" >> $GITHUB_OUTPUT + else + echo "TAG=$NEW_TAG-$FOUND" >> $GITHUB_OUTPUT + fi - name: Get registration token id: getRegToken run: | - curl -X POST -H \"Accept: application/vnd.github+json\" -H 'Authorization: token ${{ secrets.GITHUB_TOKEN }}' https://api.github.com/repos/ton-blockchain/ton/actions/runners/registration-token + curl -X POST -H \"Accept: application/vnd.github+json\" -H 'Authorization: token ${{ secrets.GITHUB_TOKEN }}' https://api.github.com/repos/ton-blockchain/ton/actions/runners/registration-token - name: Create release id: create_release @@ -131,16 +140,16 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: - tag_name: v${{ steps.date.outputs.date }} - release_name: v${{ steps.date.outputs.date }} + tag_name: ${{ steps.tag.outputs.TAG }} + release_name: TON ${{ steps.tag.outputs.TAG }} body: | ${{ steps.read_release.outputs.CHANGELOG_BODY }} draft: false prerelease: false -# upload + # upload -# win + # win - name: Upload Windows 2019 artifacts uses: svenstaro/upload-release-action@v2 @@ -148,7 +157,7 @@ jobs: repo_token: ${{ secrets.GITHUB_TOKEN }} file: artifacts/ton-win-binaries.zip asset_name: ton-win-x86-64.zip - tag: v${{ steps.date.outputs.date }} + tag: ${{ steps.tag.outputs.TAG }} - name: Upload Windows 2019 single artifact - fift uses: svenstaro/upload-release-action@v2 @@ -156,7 +165,7 @@ jobs: repo_token: ${{ secrets.GITHUB_TOKEN }} file: artifacts/ton-win-binaries/fift.exe asset_name: fift.exe - tag: v${{ steps.date.outputs.date }} + tag: ${{ steps.tag.outputs.TAG }} - name: Upload Windows 2019 single artifact - func uses: svenstaro/upload-release-action@v2 @@ -164,7 +173,7 @@ jobs: repo_token: ${{ secrets.GITHUB_TOKEN }} file: artifacts/ton-win-binaries/func.exe asset_name: func.exe - tag: v${{ steps.date.outputs.date }} + tag: ${{ steps.tag.outputs.TAG }} - name: Upload Windows 2019 single artifact - lite-client uses: svenstaro/upload-release-action@v2 @@ -172,7 +181,7 @@ jobs: repo_token: ${{ secrets.GITHUB_TOKEN }} file: artifacts/ton-win-binaries/lite-client.exe asset_name: lite-client.exe - tag: v${{ steps.date.outputs.date }} + tag: ${{ steps.tag.outputs.TAG }} - name: Upload Windows 2019 single artifact - rldp-http-proxy uses: svenstaro/upload-release-action@v2 @@ -180,7 +189,7 @@ jobs: repo_token: ${{ secrets.GITHUB_TOKEN }} file: artifacts/ton-win-binaries/rldp-http-proxy.exe asset_name: rldp-http-proxy.exe - tag: v${{ steps.date.outputs.date }} + tag: ${{ steps.tag.outputs.TAG }} - name: Upload Windows 2019 single artifact - http-proxy uses: svenstaro/upload-release-action@v2 @@ -188,7 +197,7 @@ jobs: repo_token: ${{ secrets.GITHUB_TOKEN }} file: artifacts/ton-win-binaries/http-proxy.exe asset_name: http-proxy.exe - tag: v${{ steps.date.outputs.date }} + tag: ${{ steps.tag.outputs.TAG }} - name: Upload Windows 2019 single artifact - storage-daemon-cli uses: svenstaro/upload-release-action@v2 @@ -196,7 +205,7 @@ jobs: repo_token: ${{ secrets.GITHUB_TOKEN }} file: artifacts/ton-win-binaries/storage-daemon-cli.exe asset_name: storage-daemon-cli.exe - tag: v${{ steps.date.outputs.date }} + tag: ${{ steps.tag.outputs.TAG }} - name: Upload Windows 2019 single artifact - tonlibjson uses: svenstaro/upload-release-action@v2 @@ -204,7 +213,7 @@ jobs: repo_token: ${{ secrets.GITHUB_TOKEN }} file: artifacts/ton-win-binaries/tonlibjson.dll asset_name: tonlibjson.dll - tag: v${{ steps.date.outputs.date }} + tag: ${{ steps.tag.outputs.TAG }} - name: Upload Windows 2019 single artifact - tonlib-cli uses: svenstaro/upload-release-action@v2 @@ -212,9 +221,9 @@ jobs: repo_token: ${{ secrets.GITHUB_TOKEN }} file: artifacts/ton-win-binaries/tonlib-cli.exe asset_name: tonlib-cli.exe - tag: v${{ steps.date.outputs.date }} + tag: ${{ steps.tag.outputs.TAG }} -# mac x86-64 + # mac x86-64 - name: Upload Mac x86-64 artifacts uses: svenstaro/upload-release-action@v2 @@ -222,7 +231,7 @@ jobs: repo_token: ${{ secrets.GITHUB_TOKEN }} file: artifacts/ton-x86_64-macos-binaries.zip asset_name: ton-mac-x86-64.zip - tag: v${{ steps.date.outputs.date }} + tag: ${{ steps.tag.outputs.TAG }} - name: Upload Mac x86-64 single artifact - fift uses: svenstaro/upload-release-action@v2 @@ -230,7 +239,7 @@ jobs: repo_token: ${{ secrets.GITHUB_TOKEN }} file: artifacts/ton-x86_64-macos-binaries/fift asset_name: fift-mac-x86-64 - tag: v${{ steps.date.outputs.date }} + tag: ${{ steps.tag.outputs.TAG }} - name: Upload Mac x86-64 single artifact - func uses: svenstaro/upload-release-action@v2 @@ -238,7 +247,7 @@ jobs: repo_token: ${{ secrets.GITHUB_TOKEN }} file: artifacts/ton-x86_64-macos-binaries/func asset_name: func-mac-x86-64 - tag: v${{ steps.date.outputs.date }} + tag: ${{ steps.tag.outputs.TAG }} - name: Upload Mac x86-64 single artifact - lite-client uses: svenstaro/upload-release-action@v2 @@ -246,7 +255,7 @@ jobs: repo_token: ${{ secrets.GITHUB_TOKEN }} file: artifacts/ton-x86_64-macos-binaries/lite-client asset_name: lite-client-mac-x86-64 - tag: v${{ steps.date.outputs.date }} + tag: ${{ steps.tag.outputs.TAG }} - name: Upload Mac x86-64 single artifact - rldp-http-proxy uses: svenstaro/upload-release-action@v2 @@ -254,7 +263,7 @@ jobs: repo_token: ${{ secrets.GITHUB_TOKEN }} file: artifacts/ton-x86_64-macos-binaries/rldp-http-proxy asset_name: rldp-http-proxy-mac-x86-64 - tag: v${{ steps.date.outputs.date }} + tag: ${{ steps.tag.outputs.TAG }} - name: Upload Mac x86-64 single artifact - http-proxy uses: svenstaro/upload-release-action@v2 @@ -262,7 +271,7 @@ jobs: repo_token: ${{ secrets.GITHUB_TOKEN }} file: artifacts/ton-x86_64-macos-binaries/http-proxy asset_name: http-proxy-mac-x86-64 - tag: v${{ steps.date.outputs.date }} + tag: ${{ steps.tag.outputs.TAG }} - name: Upload Mac x86-64 single artifact - storage-daemon-cli uses: svenstaro/upload-release-action@v2 @@ -270,7 +279,7 @@ jobs: repo_token: ${{ secrets.GITHUB_TOKEN }} file: artifacts/ton-x86_64-macos-binaries/storage-daemon-cli asset_name: storage-daemon-cli-mac-x86-64 - tag: v${{ steps.date.outputs.date }} + tag: ${{ steps.tag.outputs.TAG }} - name: Upload Mac x86-64 single artifact - tonlibjson uses: svenstaro/upload-release-action@v2 @@ -278,7 +287,7 @@ jobs: repo_token: ${{ secrets.GITHUB_TOKEN }} file: artifacts/ton-x86_64-macos-binaries/libtonlibjson.dylib asset_name: tonlibjson-mac-x86-64.dylib - tag: v${{ steps.date.outputs.date }} + tag: ${{ steps.tag.outputs.TAG }} - name: Upload Mac x86-64 single artifact - tonlib-cli uses: svenstaro/upload-release-action@v2 @@ -286,9 +295,9 @@ jobs: repo_token: ${{ secrets.GITHUB_TOKEN }} file: artifacts/ton-x86_64-macos-binaries/tonlib-cli asset_name: tonlib-cli-mac-x86-64 - tag: v${{ steps.date.outputs.date }} + tag: ${{ steps.tag.outputs.TAG }} -# linux x86-64 + # linux x86-64 - name: Upload Linux x86-64 artifacts uses: svenstaro/upload-release-action@v2 @@ -296,7 +305,7 @@ jobs: repo_token: ${{ secrets.GITHUB_TOKEN }} file: artifacts/ton-x86_64-linux-binaries.zip asset_name: ton-linux-x86_64.zip - tag: v${{ steps.date.outputs.date }} + tag: ${{ steps.tag.outputs.TAG }} - name: Upload Linux x86-64 single artifact - fift uses: svenstaro/upload-release-action@v2 @@ -304,7 +313,7 @@ jobs: repo_token: ${{ secrets.GITHUB_TOKEN }} file: artifacts/ton-x86_64-linux-binaries/fift asset_name: fift-linux-x86_64 - tag: v${{ steps.date.outputs.date }} + tag: ${{ steps.tag.outputs.TAG }} - name: Upload Linux x86-64 single artifact - func uses: svenstaro/upload-release-action@v2 @@ -312,7 +321,7 @@ jobs: repo_token: ${{ secrets.GITHUB_TOKEN }} file: artifacts/ton-x86_64-linux-binaries/func asset_name: func-linux-x86_64 - tag: v${{ steps.date.outputs.date }} + tag: ${{ steps.tag.outputs.TAG }} - name: Upload Linux x86-64 single artifact - lite-client uses: svenstaro/upload-release-action@v2 @@ -320,7 +329,7 @@ jobs: repo_token: ${{ secrets.GITHUB_TOKEN }} file: artifacts/ton-x86_64-linux-binaries/lite-client asset_name: lite-client-linux-x86_64 - tag: v${{ steps.date.outputs.date }} + tag: ${{ steps.tag.outputs.TAG }} - name: Upload Linux x86-64 single artifact - rldp-http-proxy uses: svenstaro/upload-release-action@v2 @@ -328,7 +337,7 @@ jobs: repo_token: ${{ secrets.GITHUB_TOKEN }} file: artifacts/ton-x86_64-linux-binaries/rldp-http-proxy asset_name: rldp-http-proxy-linux-x86_64 - tag: v${{ steps.date.outputs.date }} + tag: ${{ steps.tag.outputs.TAG }} - name: Upload Linux x86-64 single artifact - http-proxy uses: svenstaro/upload-release-action@v2 @@ -336,7 +345,7 @@ jobs: repo_token: ${{ secrets.GITHUB_TOKEN }} file: artifacts/ton-x86_64-linux-binaries/http-proxy asset_name: http-proxy-linux-x86_64 - tag: v${{ steps.date.outputs.date }} + tag: ${{ steps.tag.outputs.TAG }} - name: Upload Linux x86-64 single artifact - storage-daemon-cli uses: svenstaro/upload-release-action@v2 @@ -344,7 +353,7 @@ jobs: repo_token: ${{ secrets.GITHUB_TOKEN }} file: artifacts/ton-x86_64-linux-binaries/storage-daemon-cli asset_name: storage-daemon-cli-linux-x86_64 - tag: v${{ steps.date.outputs.date }} + tag: ${{ steps.tag.outputs.TAG }} - name: Upload Linux x86-64 single artifact - tonlibjson uses: svenstaro/upload-release-action@v2 @@ -352,7 +361,7 @@ jobs: repo_token: ${{ secrets.GITHUB_TOKEN }} file: artifacts/ton-x86_64-linux-binaries/libtonlibjson.so.0.5 asset_name: tonlibjson-linux-x86_64.so - tag: v${{ steps.date.outputs.date }} + tag: ${{ steps.tag.outputs.TAG }} - name: Upload Linux x86-64 single artifact - tonlib-cli uses: svenstaro/upload-release-action@v2 @@ -360,23 +369,23 @@ jobs: repo_token: ${{ secrets.GITHUB_TOKEN }} file: artifacts/ton-x86_64-linux-binaries/tonlib-cli asset_name: tonlib-cli-linux-x86_64 - tag: v${{ steps.date.outputs.date }} - -# - name: Upload Linux arm64 artifacts -# uses: svenstaro/upload-release-action@v2 -# with: -# repo_token: ${{ secrets.GITHUB_TOKEN }} -# file: artifacts/ton-aarch64-linux-binaries.zip -# asset_name: ton-linux-arm64.zip -# tag: v${{ steps.date.outputs.date }} -# -# - name: Upload Mac arm64 artifacts -# uses: svenstaro/upload-release-action@v2 -# with: -# repo_token: ${{ secrets.GITHUB_TOKEN }} -# file: artifacts/ton-aarch64-macos-binaries -# asset_name: ton-mac-arm64.zip -# tag: v${{ steps.date.outputs.date }} + tag: ${{ steps.tag.outputs.TAG }} + + # - name: Upload Linux arm64 artifacts + # uses: svenstaro/upload-release-action@v2 + # with: + # repo_token: ${{ secrets.GITHUB_TOKEN }} + # file: artifacts/ton-aarch64-linux-binaries.zip + # asset_name: ton-linux-arm64.zip + # tag: ${{ steps.tag.outputs.TAG }} + # + # - name: Upload Mac arm64 artifacts + # uses: svenstaro/upload-release-action@v2 + # with: + # repo_token: ${{ secrets.GITHUB_TOKEN }} + # file: artifacts/ton-aarch64-macos-binaries + # asset_name: ton-mac-arm64.zip + # tag: ${{ steps.tag.outputs.TAG }} - name: Upload WASM artifacts uses: svenstaro/upload-release-action@v2 @@ -384,4 +393,4 @@ jobs: repo_token: ${{ secrets.GITHUB_TOKEN }} file: artifacts/ton-wasm-binaries.zip asset_name: ton-wasm-binaries.zip - tag: v${{ steps.date.outputs.date }} + tag: ${{ steps.tag.outputs.TAG }} diff --git a/recent_changelog.md b/recent_changelog.md new file mode 100644 index 000000000..592976697 --- /dev/null +++ b/recent_changelog.md @@ -0,0 +1,6 @@ +## 03.2023 Update +1. Improvement of ADNL connection stability +2. Transaction emulator support and getAccountStateByTransaction method +3. Fixes of typos, undefined behavior and timer warnings +4. Handling incorrect integer literal values in funC; funC version bumped to 0.4.2 +5. FunC Mathlib \ No newline at end of file From 5abfe2337e26e214dc04d2e7840fb9b2e640e530 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Wed, 10 May 2023 12:57:57 +0300 Subject: [PATCH 17/20] Catchain improvements (#698) * Fix "sgn" in fift; fix marking infinite loops as noterurn in func * TON-P1-1: Remove unused catchain queries * TON-P1-15: Avoid synchronization with self * TON-P1-3, TON-P1-17: Disallow more than one candidate per src per round (to prevent flood), add checks to process_broadcast * TON-P1-10: Fix fast/slow attempts * TON-P1-14: Add named constants * TON-P1-18, TON-P1-19: Alloc temporary memory in the same way as persistent memory * TON-P1-20: Add comment to choose_blocks_to_approve * TON-P1-16: Avoid creating two catchain blocks on need_new_block * TON-P1-8: Add some validation to validator-engine parameters * TON-P1-6: Don't allow sending the same block many times Many requests for the same block are not unusual (however, there's no need to answer them all) * TON-P1-2: Enable prohibiting dependencies from blamed nodes (2.7.5 in CatChain doc), fix processing blame proofs * Best practices bp-6: Fix incorrect warning bp-7: Remove unused code bp-8: Bring back PerfWarningTimer logging (only when no callback) bp-9: Remove unnecessary condition bp-11: Remove commented-out code bp-13: Divide code in validator-session-state Adherence to Specification: Fix typo --- catchain/catchain-receiver-interface.h | 1 + catchain/catchain-receiver-source.cpp | 27 +- catchain/catchain-receiver-source.h | 3 + catchain/catchain-receiver-source.hpp | 7 + catchain/catchain-receiver.cpp | 154 ++--- catchain/catchain-receiver.h | 2 +- catchain/catchain-receiver.hpp | 17 +- catchain/catchain.cpp | 15 +- crypto/fift/words.cpp | 2 +- crypto/func/analyzer.cpp | 2 +- doc/catchain.tex | 2 +- tdutils/td/utils/Timer.cpp | 9 +- tdutils/td/utils/Timer.h | 2 +- tl/generate/scheme/ton_api.tl | 4 - tl/generate/scheme/ton_api.tlo | Bin 86540 -> 86116 bytes validator-engine/validator-engine.cpp | 80 ++- validator-session/CMakeLists.txt | 3 +- validator-session/persistent-vector.h | 7 - .../validator-session-description.cpp | 81 ++- .../validator-session-description.hpp | 32 +- .../validator-session-round-attempt-state.cpp | 486 ++++++++++++++ .../validator-session-round-attempt-state.h | 620 ++++++++++++++++++ validator-session/validator-session-state.cpp | 473 +------------ validator-session/validator-session-state.h | 590 +---------------- validator-session/validator-session-types.h | 6 + validator-session/validator-session.cpp | 70 +- validator-session/validator-session.hpp | 11 +- 27 files changed, 1410 insertions(+), 1296 deletions(-) create mode 100644 validator-session/validator-session-round-attempt-state.cpp create mode 100644 validator-session/validator-session-round-attempt-state.h diff --git a/catchain/catchain-receiver-interface.h b/catchain/catchain-receiver-interface.h index c8f1ef66d..bc02832ae 100644 --- a/catchain/catchain-receiver-interface.h +++ b/catchain/catchain-receiver-interface.h @@ -52,6 +52,7 @@ class CatChainReceiverInterface : public td::actor::Actor { td::BufferSlice query, td::uint64 max_answer_size, td::actor::ActorId via) = 0; virtual void send_custom_message_data(const PublicKeyHash &dst, td::BufferSlice query) = 0; + virtual void on_blame_processed(td::uint32 source_id) = 0; virtual void destroy() = 0; diff --git a/catchain/catchain-receiver-source.cpp b/catchain/catchain-receiver-source.cpp index 6a9e0777c..e758b3359 100644 --- a/catchain/catchain-receiver-source.cpp +++ b/catchain/catchain-receiver-source.cpp @@ -60,15 +60,15 @@ td::Result> CatChainReceiverSource::crea void CatChainReceiverSourceImpl::blame(td::uint32 fork, CatChainBlockHeight height) { blame(); - if (!blamed_heights_.empty()) { - if (blamed_heights_.size() <= fork) { - blamed_heights_.resize(fork + 1, 0); - } - if (blamed_heights_[fork] == 0 || blamed_heights_[fork] > height) { - VLOG(CATCHAIN_INFO) << this << ": blamed at " << fork << " " << height; - blamed_heights_[fork] = height; - } + // if (!blamed_heights_.empty()) { + if (blamed_heights_.size() <= fork) { + blamed_heights_.resize(fork + 1, 0); + } + if (blamed_heights_[fork] == 0 || blamed_heights_[fork] > height) { + VLOG(CATCHAIN_INFO) << this << ": blamed at " << fork << " " << height; + blamed_heights_[fork] = height; } + // } } void CatChainReceiverSourceImpl::blame() { @@ -144,7 +144,7 @@ void CatChainReceiverSourceImpl::on_new_block(CatChainReceivedBlock *block) { on_found_fork_proof(create_serialize_tl_object(block->export_tl_dep(), it->second->export_tl_dep()) .as_slice()); - chain_->add_prepared_event(fork_proof()); + chain_->on_found_fork_proof(id_, fork_proof()); } blame(); return; @@ -162,6 +162,15 @@ void CatChainReceiverSourceImpl::on_found_fork_proof(const td::Slice &proof) { } } +bool CatChainReceiverSourceImpl::allow_send_block(CatChainBlockHash hash) { + td::uint32 count = ++block_requests_count_[hash]; + if (count > MAX_BLOCK_REQUESTS) { + VLOG(CATCHAIN_INFO) << this << ": node requested block " << hash << " " << count << " times"; + return false; + } + return true; +} + } // namespace catchain } // namespace ton diff --git a/catchain/catchain-receiver-source.h b/catchain/catchain-receiver-source.h index e805c2a55..55035e779 100644 --- a/catchain/catchain-receiver-source.h +++ b/catchain/catchain-receiver-source.h @@ -61,6 +61,9 @@ class CatChainReceiverSource { virtual td::BufferSlice fork_proof() const = 0; virtual bool fork_is_found() const = 0; + // One block can be sent to one node only a limited number of times to prevent DoS + virtual bool allow_send_block(CatChainBlockHash hash) = 0; + static td::Result> create(CatChainReceiver *chain, PublicKey pub_key, adnl::AdnlNodeIdShort adnl_id, td::uint32 id); diff --git a/catchain/catchain-receiver-source.hpp b/catchain/catchain-receiver-source.hpp index 0785d8c2f..cf08c4219 100644 --- a/catchain/catchain-receiver-source.hpp +++ b/catchain/catchain-receiver-source.hpp @@ -111,6 +111,8 @@ class CatChainReceiverSourceImpl : public CatChainReceiverSource { return chain_; } + bool allow_send_block(CatChainBlockHash hash) override; + CatChainReceiverSourceImpl(CatChainReceiver *chain, PublicKey source, adnl::AdnlNodeIdShort adnl_id, td::uint32 id); private: @@ -130,6 +132,11 @@ class CatChainReceiverSourceImpl : public CatChainReceiverSource { CatChainBlockHeight delivered_height_ = 0; CatChainBlockHeight received_height_ = 0; + + std::map block_requests_count_; + // One block can be sent to one node up to 5 times + + static const td::uint32 MAX_BLOCK_REQUESTS = 5; }; } // namespace catchain diff --git a/catchain/catchain-receiver.cpp b/catchain/catchain-receiver.cpp index 488fbb9d5..482cfb43c 100644 --- a/catchain/catchain-receiver.cpp +++ b/catchain/catchain-receiver.cpp @@ -168,13 +168,6 @@ void CatChainReceiverImpl::receive_message_from_overlay(adnl::AdnlNodeIdShort sr return; } - /*auto S = get_source_by_hash(src); - CHECK(S != nullptr); - - if (S->blamed()) { - VLOG(CATCHAIN_INFO) << this << ": dropping block update from blamed source " << src; - return; - }*/ if (data.size() > opts_.max_serialized_block_size) { VLOG(CATCHAIN_WARNING) << this << ": dropping broken block from " << src << ": too big (size=" << data.size() << ", limit=" << opts_.max_serialized_block_size << ")"; @@ -197,19 +190,6 @@ void CatChainReceiverImpl::receive_broadcast_from_overlay(const PublicKeyHash &s callback_->on_broadcast(src, std::move(data)); } -/*void CatChainReceiverImpl::send_block(const PublicKeyHash &src, tl_object_ptr block, - td::BufferSlice payload) { - CHECK(read_db_); - CHECK(src == local_id_); - - validate_block_sync(block, payload.as_slice()).ensure(); - auto B = create_block(std::move(block), td::SharedSlice{payload.as_slice()}); - CHECK(B != nullptr); - - run_scheduler(); - CHECK(B->delivered()); -}*/ - CatChainReceivedBlock *CatChainReceiverImpl::create_block(tl_object_ptr block, td::SharedSlice payload) { if (block->height_ == 0) { @@ -267,21 +247,16 @@ td::Status CatChainReceiverImpl::validate_block_sync(const tl_object_ptr &block, const td::Slice &payload) const { - //LOG(INFO) << ton_api::to_string(block); TRY_STATUS_PREFIX(CatChainReceivedBlock::pre_validate_block(this, block, payload), "failed to validate block: "); + // After pre_validate_block, block->height_ > 0 + auto id = CatChainReceivedBlock::block_id(this, block, payload); + td::BufferSlice B = serialize_tl_object(id, true); - if (block->height_ > 0) { - auto id = CatChainReceivedBlock::block_id(this, block, payload); - td::BufferSlice B = serialize_tl_object(id, true); - - CatChainReceiverSource *S = get_source_by_hash(PublicKeyHash{id->src_}); - CHECK(S != nullptr); - Encryptor *E = S->get_encryptor_sync(); - CHECK(E != nullptr); - return E->check_signature(B.as_slice(), block->signature_.as_slice()); - } else { - return td::Status::OK(); - } + CatChainReceiverSource *S = get_source_by_hash(PublicKeyHash{id->src_}); + CHECK(S != nullptr); + Encryptor *E = S->get_encryptor_sync(); + CHECK(E != nullptr); + return E->check_signature(B.as_slice(), block->signature_.as_slice()); } void CatChainReceiverImpl::run_scheduler() { @@ -515,7 +490,6 @@ CatChainReceiverImpl::CatChainReceiverImpl(std::unique_ptr callback, } CHECK(local_idx_ != static_cast(ids.size())); - //std::sort(short_ids.begin(), short_ids.end()); auto F = create_tl_object(unique_hash, std::move(short_ids)); overlay_full_id_ = overlay::OverlayIdFull{serialize_tl_object(F, true)}; @@ -527,6 +501,8 @@ CatChainReceiverImpl::CatChainReceiverImpl(std::unique_ptr callback, blocks_[root_block_->get_hash()] = std::move(R); last_sent_block_ = root_block_; + blame_processed_.resize(sources_.size(), false); + choose_neighbours(); } @@ -704,7 +680,6 @@ void CatChainReceiverImpl::receive_query_from_overlay(adnl::AdnlNodeIdShort src, auto F = fetch_tl_object(data.clone(), true); if (F.is_error()) { callback_->on_custom_query(get_source_by_adnl_id(src)->get_hash(), std::move(data), std::move(promise)); - //LOG(WARNING) << this << ": unknown query from " << src; return; } auto f = F.move_as_ok(); @@ -717,68 +692,15 @@ void CatChainReceiverImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::cat if (it == blocks_.end() || it->second->get_height() == 0 || !it->second->initialized()) { promise.set_value(serialize_tl_object(create_tl_object(), true)); } else { - promise.set_value(serialize_tl_object(create_tl_object(it->second->export_tl()), - true, it->second->get_payload().as_slice())); - } -} - -void CatChainReceiverImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::catchain_getBlocks query, - td::Promise promise) { - if (query.blocks_.size() > MAX_QUERY_BLOCKS) { - promise.set_error(td::Status::Error(ErrorCode::protoviolation, "too many blocks")); - return; - } - td::int32 cnt = 0; - for (const CatChainBlockHash &b : query.blocks_) { - auto it = blocks_.find(b); - if (it != blocks_.end() && it->second->get_height() > 0) { - auto block = create_tl_object(it->second->export_tl()); - CHECK(!it->second->get_payload().empty()); - td::BufferSlice B = serialize_tl_object(block, true, it->second->get_payload().clone()); - CHECK(B.size() <= opts_.max_serialized_block_size); - td::actor::send_closure(overlay_manager_, &overlay::Overlays::send_message, src, - get_source(local_idx_)->get_adnl_id(), overlay_id_, std::move(B)); - cnt++; + CatChainReceiverSource *S = get_source_by_adnl_id(src); + CHECK(S != nullptr); + if (S->allow_send_block(it->second->get_hash())) { + promise.set_value(serialize_tl_object(create_tl_object(it->second->export_tl()), + true, it->second->get_payload().as_slice())); + } else { + promise.set_error(td::Status::Error("block was requested too many times")); } } - promise.set_value(serialize_tl_object(create_tl_object(cnt), true)); -} - -void CatChainReceiverImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::catchain_getBlockHistory query, - td::Promise promise) { - int64_t h = query.height_; - if (h <= 0) { - promise.set_error(td::Status::Error(ErrorCode::protoviolation, "not-positive height")); - return; - } - if (h > MAX_QUERY_HEIGHT) { - h = MAX_QUERY_HEIGHT; - } - std::set s{query.stop_if_.begin(), query.stop_if_.end()}; - - CatChainReceivedBlock *B = get_block(query.block_); - if (B == nullptr) { - promise.set_value(serialize_tl_object(create_tl_object(0), true)); - return; - } - if (static_cast(h) > B->get_height()) { - h = B->get_height(); - } - td::uint32 cnt = 0; - while (h-- > 0) { - if (s.find(B->get_hash()) != s.end()) { - break; - } - auto block = create_tl_object(B->export_tl()); - CHECK(!B->get_payload().empty()); - td::BufferSlice BB = serialize_tl_object(block, true, B->get_payload().as_slice()); - CHECK(BB.size() <= opts_.max_serialized_block_size); - td::actor::send_closure(overlay_manager_, &overlay::Overlays::send_message, src, - get_source(local_idx_)->get_adnl_id(), overlay_id_, std::move(BB)); - B = B->get_prev(); - cnt++; - } - promise.set_value(serialize_tl_object(create_tl_object(cnt), true)); } void CatChainReceiverImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::catchain_getDifference query, @@ -832,6 +754,8 @@ void CatChainReceiverImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::cat } } CHECK(right > 0); + CatChainReceiverSource *S0 = get_source_by_adnl_id(src); + CHECK(S0 != nullptr); for (td::uint32 i = 0; i < get_sources_cnt(); i++) { if (vt[i] >= 0 && my_vt[i] > vt[i]) { CatChainReceiverSource *S = get_source(i); @@ -839,12 +763,14 @@ void CatChainReceiverImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::cat while (t-- > 0) { CatChainReceivedBlock *M = S->get_block(++vt[i]); CHECK(M != nullptr); - auto block = create_tl_object(M->export_tl()); - CHECK(!M->get_payload().empty()); - td::BufferSlice BB = serialize_tl_object(block, true, M->get_payload().as_slice()); - CHECK(BB.size() <= opts_.max_serialized_block_size); - td::actor::send_closure(overlay_manager_, &overlay::Overlays::send_message, src, - get_source(local_idx_)->get_adnl_id(), overlay_id_, std::move(BB)); + if (S0->allow_send_block(M->get_hash())) { + auto block = create_tl_object(M->export_tl()); + CHECK(!M->get_payload().empty()); + td::BufferSlice BB = serialize_tl_object(block, true, M->get_payload().as_slice()); + CHECK(BB.size() <= opts_.max_serialized_block_size); + td::actor::send_closure(overlay_manager_, &overlay::Overlays::send_message, src, + get_source(local_idx_)->get_adnl_id(), overlay_id_, std::move(BB)); + } } } } @@ -1031,10 +957,15 @@ void CatChainReceiverImpl::written_unsafe_root_block(CatChainReceivedBlock *bloc void CatChainReceiverImpl::alarm() { alarm_timestamp() = td::Timestamp::never(); - if (next_sync_ && next_sync_.is_in_past()) { + if (next_sync_ && next_sync_.is_in_past() && get_sources_cnt() > 1) { next_sync_ = td::Timestamp::in(td::Random::fast(SYNC_INTERVAL_MIN, SYNC_INTERVAL_MAX)); for (unsigned i = 0; i < SYNC_ITERATIONS; i++) { - CatChainReceiverSource *S = get_source(td::Random::fast(0, static_cast(get_sources_cnt()) - 1)); + auto idx = td::Random::fast(1, static_cast(get_sources_cnt()) - 1); + if (idx == static_cast(local_idx_)) { + idx = 0; + } + // idx is a random number in [0, get_sources_cnt-1] not equal to local_idx + CatChainReceiverSource *S = get_source(idx); CHECK(S != nullptr); if (!S->blamed()) { synchronize_with(S); @@ -1117,6 +1048,23 @@ static void destroy_db(const std::string& name, td::uint32 attempt) { } } +void CatChainReceiverImpl::on_found_fork_proof(td::uint32 source_id, td::BufferSlice data) { + if (blame_processed_[source_id]) { + add_block(std::move(data), std::vector()); + } else { + pending_fork_proofs_[source_id] = std::move(data); + } +} + +void CatChainReceiverImpl::on_blame_processed(td::uint32 source_id) { + blame_processed_[source_id] = true; + auto it = pending_fork_proofs_.find(source_id); + if (it != pending_fork_proofs_.end()) { + add_block(std::move(it->second), std::vector()); + pending_fork_proofs_.erase(it); + } +} + void CatChainReceiverImpl::destroy() { auto name = db_root_ + "/catchainreceiver" + db_suffix_ + td::base64url_encode(as_slice(incarnation_)); delay_action([name]() { destroy_db(name, 0); }, td::Timestamp::in(DESTROY_DB_DELAY)); diff --git a/catchain/catchain-receiver.h b/catchain/catchain-receiver.h index 2c959fbc2..75c873511 100644 --- a/catchain/catchain-receiver.h +++ b/catchain/catchain-receiver.h @@ -56,7 +56,7 @@ class CatChainReceiver : public CatChainReceiverInterface { virtual void run_block(CatChainReceivedBlock *block) = 0; virtual void deliver_block(CatChainReceivedBlock *block) = 0; virtual td::uint32 add_fork() = 0; - virtual void add_prepared_event(td::BufferSlice data) = 0; + virtual void on_found_fork_proof(td::uint32 source_id, td::BufferSlice data) = 0; virtual void on_blame(td::uint32 source_id) = 0; virtual const CatChainOptions &opts() const = 0; diff --git a/catchain/catchain-receiver.hpp b/catchain/catchain-receiver.hpp index 5da4001c1..5c4d37648 100644 --- a/catchain/catchain-receiver.hpp +++ b/catchain/catchain-receiver.hpp @@ -39,9 +39,6 @@ class CatChainReceiverImpl final : public CatChainReceiver { return PrintId{incarnation_, local_id_}; } - void add_prepared_event(td::BufferSlice data) override { - add_block(std::move(data), std::vector()); - } CatChainSessionId get_incarnation() const override { return incarnation_; } @@ -73,15 +70,10 @@ class CatChainReceiverImpl final : public CatChainReceiver { void receive_query_from_overlay(adnl::AdnlNodeIdShort src, td::BufferSlice data, td::Promise promise); void process_query(adnl::AdnlNodeIdShort src, ton_api::catchain_getBlock query, td::Promise promise); - void process_query(adnl::AdnlNodeIdShort src, ton_api::catchain_getBlocks query, - td::Promise promise); - void process_query(adnl::AdnlNodeIdShort src, ton_api::catchain_getBlockHistory query, - td::Promise promise); void process_query(adnl::AdnlNodeIdShort src, ton_api::catchain_getDifference query, td::Promise promise); template void process_query(adnl::AdnlNodeIdShort src, const T &query, td::Promise promise) { - //LOG(WARNING) << this << ": unknown query from " << src; callback_->on_custom_query(get_source_by_adnl_id(src)->get_hash(), serialize_tl_object(&query, true), std::move(promise)); } @@ -89,7 +81,6 @@ class CatChainReceiverImpl final : public CatChainReceiver { void receive_block(adnl::AdnlNodeIdShort src, tl_object_ptr block, td::BufferSlice payload); void receive_block_answer(adnl::AdnlNodeIdShort src, td::BufferSlice); - //void send_block(const PublicKeyHash &src, tl_object_ptr block, td::BufferSlice payload); CatChainReceivedBlock *create_block(tl_object_ptr block, td::SharedSlice payload) override; CatChainReceivedBlock *create_block(tl_object_ptr block) override; @@ -117,6 +108,8 @@ class CatChainReceiverImpl final : public CatChainReceiver { void on_blame(td::uint32 src) override { callback_->blame(src); } + void on_found_fork_proof(td::uint32 source_id, td::BufferSlice data) override; + void on_blame_processed(td::uint32 source_id) override; const CatChainOptions &opts() const override { return opts_; } @@ -204,9 +197,6 @@ class CatChainReceiverImpl final : public CatChainReceiver { std::vector neighbours_; - //std::queue> events_; - //std::queue raw_events_; - td::actor::ActorId keyring_; td::actor::ActorId adnl_; td::actor::ActorId overlay_manager_; @@ -231,6 +221,9 @@ class CatChainReceiverImpl final : public CatChainReceiver { bool started_{false}; std::list to_run_; + + std::vector blame_processed_; + std::map pending_fork_proofs_; }; } // namespace catchain diff --git a/catchain/catchain.cpp b/catchain/catchain.cpp index 001840fd9..53ae16b52 100644 --- a/catchain/catchain.cpp +++ b/catchain/catchain.cpp @@ -108,11 +108,12 @@ void CatChainImpl::need_new_block(td::Timestamp t) { if (!receiver_started_) { return; } - if (!force_process_) { + if (!force_process_ || !active_process_) { VLOG(CATCHAIN_INFO) << this << ": forcing creation of new block"; } - force_process_ = true; - if (!active_process_) { + if (active_process_) { + force_process_ = true; + } else { alarm_timestamp().relax(t); } } @@ -197,6 +198,14 @@ void CatChainImpl::on_blame(td::uint32 src_id) { } } } + for (td::uint32 i = 0; i < process_deps_.size(); ++i) { + if (blocks_[process_deps_[i]]->source() == src_id) { + process_deps_[i] = process_deps_.back(); + process_deps_.pop_back(); + --i; + } + } + td::actor::send_closure(receiver_, &CatChainReceiverInterface::on_blame_processed, src_id); } void CatChainImpl::on_custom_query(const PublicKeyHash &src, td::BufferSlice data, td::Promise promise) { diff --git a/crypto/fift/words.cpp b/crypto/fift/words.cpp index 964b43288..b5e6dbd69 100644 --- a/crypto/fift/words.cpp +++ b/crypto/fift/words.cpp @@ -235,7 +235,7 @@ void interpret_cmp(vm::Stack& stack, const char opt[3]) { } void interpret_sgn(vm::Stack& stack, const char opt[3]) { - auto x = stack.pop_int(); + auto x = stack.pop_int_finite(); int r = x->sgn(); assert((unsigned)(r + 1) <= 2); stack.push_smallint(((const signed char*)opt)[r + 1]); diff --git a/crypto/func/analyzer.cpp b/crypto/func/analyzer.cpp index 989e9ebe9..d59230e87 100644 --- a/crypto/func/analyzer.cpp +++ b/crypto/func/analyzer.cpp @@ -894,7 +894,7 @@ bool Op::mark_noreturn() { return set_noreturn((block0->mark_noreturn() & (block1 && block1->mark_noreturn())) | next->mark_noreturn()); case _Again: block0->mark_noreturn(); - return set_noreturn(false); + return set_noreturn(true); case _Until: return set_noreturn(block0->mark_noreturn() | next->mark_noreturn()); case _While: diff --git a/doc/catchain.tex b/doc/catchain.tex index b3cae642b..b1a27164d 100644 --- a/doc/catchain.tex +++ b/doc/catchain.tex @@ -243,7 +243,7 @@ Once a fork (created by~$i$) is detected (by another process~$j$), i.e.\ $j$ learns about two different messages $m_{i,s}$ and $m'_{i,s}$ created by $i$ and having same height $s$ (usually this happens while recursively downloading dependencies of some other messages), $j$ starts ignoring~$i$ and all of its subsequent messages. They are not accepted and not broadcast further. However, messages created by~$i$ prior to the fork detection may be still downloaded if they are referred to in messages (blocks) created by processes that did not see this fork before referring to such messages created by~$i$. \nxsubpoint\emb{Accepting messages from a ``bad'' process is bad}\label{sp:no.bad.accept} -Furthermore, if process $i$ learns about a fork created by process $j$, then $i$ shows this to its neighbors by creating a new service broadcast message that contains the corresponding fork proof (cf.~\ptref{sp:fork.proofs}). Afterwards, this and all subsequent messages of $j$ cannot directly depend on any messages by the known ``bad'' producer $i$ (but they still can refer to messages from another party $k$ that directly or indirectly refer to messages of~$i$ if no fork by~$i$ was known to $k$ at the time when the referring message was created). If $j$ violates this restriction and creates messages with such invalid references, these messages will be discarded by all honest processes in the group. +Furthermore, if process $i$ learns about a fork created by process $j$, then $i$ shows this to its neighbors by creating a new service broadcast message that contains the corresponding fork proof (cf.~\ptref{sp:fork.proofs}). Afterwards, this and all subsequent messages of $i$ cannot directly depend on any messages by the known ``bad'' producer $j$ (but they still can refer to messages from another party $k$ that directly or indirectly refer to messages of~$i$ if no fork by~$i$ was known to $k$ at the time when the referring message was created). If $i$ violates this restriction and creates messages with such invalid references, these messages will be discarded by all honest processes in the group. \nxsubpoint\emb{The set of ``bad'' group members is a part of the intrinsic state}\label{sp:bad.proc.set} Each process~$i$ keeps its own copy of the set of known ``bad'' processes in the group, i.e., those processes that have created at least one fork or have violated \ptref{sp:no.bad.accept}. This set is updated by adding~$j$ into it as soon as $i$ learns about a fork created by~$j$ (or about a violation of~\ptref{sp:no.bad.accept} by $j$); after that, a callback provided by the higher-level protocol is invoked. This set is used when a new broadcast message arrives: if the sender is bad, then the message is ignored and discarded. diff --git a/tdutils/td/utils/Timer.cpp b/tdutils/td/utils/Timer.cpp index f33a30dc8..0f6a7d6ed 100644 --- a/tdutils/td/utils/Timer.cpp +++ b/tdutils/td/utils/Timer.cpp @@ -78,9 +78,12 @@ void PerfWarningTimer::reset() { return; } double duration = Time::now() - start_at_; - //LOG_IF(WARNING, duration > max_duration_) - //<< "SLOW: " << tag("name", name_) << tag("duration", format::as_time(duration)); - callback_(duration); + if (callback_) { + callback_(duration); + } else { + LOG_IF(WARNING, duration > max_duration_) + << "SLOW: " << tag("name", name_) << tag("duration", format::as_time(duration)); + } start_at_ = 0; } diff --git a/tdutils/td/utils/Timer.h b/tdutils/td/utils/Timer.h index 64f3e934d..18ea35d7c 100644 --- a/tdutils/td/utils/Timer.h +++ b/tdutils/td/utils/Timer.h @@ -46,7 +46,7 @@ class Timer { class PerfWarningTimer { public: - explicit PerfWarningTimer(string name, double max_duration = 0.1, std::function&& callback = [] (double) {}); + explicit PerfWarningTimer(string name, double max_duration = 0.1, std::function&& callback = {}); PerfWarningTimer(const PerfWarningTimer &) = delete; PerfWarningTimer &operator=(const PerfWarningTimer &) = delete; PerfWarningTimer(PerfWarningTimer &&other); diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index 66e6f4a69..346d91522 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -277,14 +277,10 @@ catchain.differenceFork left:catchain.block.dep right:catchain.block.dep = catch catchain.blockNotFound = catchain.BlockResult; catchain.blockResult block:catchain.block = catchain.BlockResult; -catchain.sent cnt:int = catchain.Sent; - ---functions--- catchain.getBlock block:int256 = catchain.BlockResult; -catchain.getBlocks blocks:(vector int256) = catchain.Sent; catchain.getDifference rt:(vector int) = catchain.Difference; -catchain.getBlockHistory block:int256 height:long stop_if:(vector int256) = catchain.Sent; //catchain.getForkDifference src:int fork:catchain.fork = catchain.ForkDifference; ---types--- diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index 8ce7a57915b72831287ea6490d8590216472546b..5b8e1c725e39ceff4097e4a654c796c9f84ff73e 100644 GIT binary patch delta 54 zcmeBK!un(ZEAOM(`c@23@NXmUdbZ6k*luVs)@(j#*si&GPV|wDjP=tKc^S8Fci?9{ G;{*T`CKL$( delta 237 zcmaE|fVF1{EAOM(`c@23@P8xkdN!f;f!}}eCMT98XC!9k=>@0el}x_JEVX$L+YK$o z+Ral9+chOnl@tS&Fff3`CNF5wVF58V3q)Vo$k;gjArIqr&2PD3DV)4 zdt?@u5LY-v8h|u`U9!E0m+^)Z072GV?*IS* diff --git a/validator-engine/validator-engine.cpp b/validator-engine/validator-engine.cpp index 02ab42f9e..65b93df71 100644 --- a/validator-engine/validator-engine.cpp +++ b/validator-engine/validator-engine.cpp @@ -1319,9 +1319,6 @@ td::Status ValidatorEngine::load_global_config() { } CHECK(mode == ton::validator::ValidatorManagerOptions::ShardCheckMode::m_validate); return true; - /*ton::ShardIdFull p{ton::basechainId, ((cc_seqno * 1ull % 4) << 62) + 1}; - auto s = ton::shard_prefix(p, 2); - return shard.is_masterchain() || ton::shard_intersects(shard, s);*/ }); if (state_ttl_ != 0) { validator_options_.write().set_state_ttl(state_ttl_); @@ -3647,34 +3644,61 @@ int main(int argc, char *argv[]) { logger_ = td::TsFileLog::create(fname.str()).move_as_ok(); td::log_interface = logger_.get(); }); - p.add_option('s', "state-ttl", "state will be gc'd after this time (in seconds) default=3600", [&](td::Slice fname) { - auto v = td::to_double(fname); - acts.push_back([&x, v]() { td::actor::send_closure(x, &ValidatorEngine::set_state_ttl, v); }); - }); - p.add_option('m', "mempool-num", "Maximal number of mempool external message", [&](td::Slice fname) { + p.add_checked_option('s', "state-ttl", "state will be gc'd after this time (in seconds) default=3600", + [&](td::Slice fname) { + auto v = td::to_double(fname); + if (v <= 0) { + return td::Status::Error("state-ttl should be positive"); + } + acts.push_back([&x, v]() { td::actor::send_closure(x, &ValidatorEngine::set_state_ttl, v); }); + return td::Status::OK(); + }); + p.add_checked_option('m', "mempool-num", "Maximal number of mempool external message", [&](td::Slice fname) { auto v = td::to_double(fname); + if (v < 0) { + return td::Status::Error("mempool-num should be non-negative"); + } acts.push_back([&x, v]() { td::actor::send_closure(x, &ValidatorEngine::set_max_mempool_num, v); }); + return td::Status::OK(); }); - p.add_option('b', "block-ttl", "blocks will be gc'd after this time (in seconds) default=7*86400", - [&](td::Slice fname) { - auto v = td::to_double(fname); - acts.push_back([&x, v]() { td::actor::send_closure(x, &ValidatorEngine::set_block_ttl, v); }); - }); - p.add_option('A', "archive-ttl", "archived blocks will be deleted after this time (in seconds) default=365*86400", - [&](td::Slice fname) { - auto v = td::to_double(fname); - acts.push_back([&x, v]() { td::actor::send_closure(x, &ValidatorEngine::set_archive_ttl, v); }); - }); - p.add_option('K', "key-proof-ttl", "key blocks will be deleted after this time (in seconds) default=365*86400*10", - [&](td::Slice fname) { - auto v = td::to_double(fname); - acts.push_back([&x, v]() { td::actor::send_closure(x, &ValidatorEngine::set_key_proof_ttl, v); }); - }); - p.add_option('S', "sync-before", "in initial sync download all blocks for last given seconds default=3600", - [&](td::Slice fname) { - auto v = td::to_double(fname); - acts.push_back([&x, v]() { td::actor::send_closure(x, &ValidatorEngine::set_sync_ttl, v); }); - }); + p.add_checked_option('b', "block-ttl", "blocks will be gc'd after this time (in seconds) default=7*86400", + [&](td::Slice fname) { + auto v = td::to_double(fname); + if (v <= 0) { + return td::Status::Error("block-ttl should be positive"); + } + acts.push_back([&x, v]() { td::actor::send_closure(x, &ValidatorEngine::set_block_ttl, v); }); + return td::Status::OK(); + }); + p.add_checked_option( + 'A', "archive-ttl", "archived blocks will be deleted after this time (in seconds) default=365*86400", + [&](td::Slice fname) { + auto v = td::to_double(fname); + if (v <= 0) { + return td::Status::Error("archive-ttl should be positive"); + } + acts.push_back([&x, v]() { td::actor::send_closure(x, &ValidatorEngine::set_archive_ttl, v); }); + return td::Status::OK(); + }); + p.add_checked_option( + 'K', "key-proof-ttl", "key blocks will be deleted after this time (in seconds) default=365*86400*10", + [&](td::Slice fname) { + auto v = td::to_double(fname); + if (v <= 0) { + return td::Status::Error("key-proof-ttl should be positive"); + } + acts.push_back([&x, v]() { td::actor::send_closure(x, &ValidatorEngine::set_key_proof_ttl, v); }); + return td::Status::OK(); + }); + p.add_checked_option('S', "sync-before", "in initial sync download all blocks for last given seconds default=3600", + [&](td::Slice fname) { + auto v = td::to_double(fname); + if (v <= 0) { + return td::Status::Error("sync-before should be positive"); + } + acts.push_back([&x, v]() { td::actor::send_closure(x, &ValidatorEngine::set_sync_ttl, v); }); + return td::Status::OK(); + }); p.add_option('T', "truncate-db", "truncate db (with specified seqno as new top masterchain block seqno)", [&](td::Slice fname) { auto v = td::to_integer(fname); diff --git a/validator-session/CMakeLists.txt b/validator-session/CMakeLists.txt index dc837219c..4931e464d 100644 --- a/validator-session/CMakeLists.txt +++ b/validator-session/CMakeLists.txt @@ -9,6 +9,7 @@ set(VALIDATOR_SESSION_SOURCE validator-session-description.cpp validator-session-state.cpp validator-session.cpp + validator-session-round-attempt-state.cpp persistent-vector.h validator-session-description.h @@ -16,7 +17,7 @@ set(VALIDATOR_SESSION_SOURCE validator-session-state.h validator-session.h validator-session.hpp -) + validator-session-round-attempt-state.h) add_library(validatorsession STATIC ${VALIDATOR_SESSION_SOURCE}) diff --git a/validator-session/persistent-vector.h b/validator-session/persistent-vector.h index 77852f80a..3491164ca 100644 --- a/validator-session/persistent-vector.h +++ b/validator-session/persistent-vector.h @@ -322,7 +322,6 @@ class CntVector : public ValidatorSessionDescription::RootObject { CHECK(idx < size()); return data_[idx]; } - //const T& at(size_t idx) const; private: const td::uint32 data_size_; @@ -546,7 +545,6 @@ class CntVector : public ValidatorSessionDescription::RootObject { CHECK(idx < max_size()); return get_bit(data_, idx); } - //const T& at(size_t idx) const; private: const td::uint32 data_size_; @@ -734,10 +732,6 @@ class CntSortedVector : public ValidatorSessionDescription::RootObject { return CntSortedVector::create(desc, std::move(v)); } - /*static const CntSortedVector* merge(ValidatorSessionDescription& desc, const CntSortedVector* l, - const CntSortedVector* r) { - return merge(desc, l, r, [](T l, T r) { return l; }); - }*/ static const CntSortedVector* push(ValidatorSessionDescription& desc, const CntSortedVector* v, T value) { if (!v) { return create(desc, std::vector{value}); @@ -795,7 +789,6 @@ class CntSortedVector : public ValidatorSessionDescription::RootObject { CHECK(idx < size()); return data_[idx]; } - //const T& at(size_t idx) const; private: const td::uint32 data_size_; diff --git a/validator-session/validator-session-description.cpp b/validator-session/validator-session-description.cpp index eb4c3a8ea..34dc90212 100644 --- a/validator-session/validator-session-description.cpp +++ b/validator-session/validator-session-description.cpp @@ -51,13 +51,6 @@ ValidatorSessionDescriptionImpl::ValidatorSessionDescriptionImpl(ValidatorSessio CHECK(it != rev_sources_.end()); self_idx_ = it->second; - pdata_temp_ptr_ = 0; - pdata_temp_size_ = 1 << 27; - pdata_temp_ = new td::uint8[pdata_temp_size_]; - - pdata_perm_size_ = 1ull << 27; - pdata_perm_ptr_ = 0; - for (auto &el : cache_) { Cached v{nullptr}; el.store(v, std::memory_order_relaxed); @@ -162,50 +155,70 @@ void ValidatorSessionDescriptionImpl::update_hash(const RootObject *obj, HashTyp } void *ValidatorSessionDescriptionImpl::alloc(size_t size, size_t align, bool temp) { - CHECK(align && !(align & (align - 1))); // align should be a power of 2 + return (temp ? mem_temp_ : mem_perm_).alloc(size, align); +} + +bool ValidatorSessionDescriptionImpl::is_persistent(const void *ptr) const { + return mem_perm_.contains(ptr); +} + +std::unique_ptr ValidatorSessionDescription::create( + ValidatorSessionOptions opts, std::vector &nodes, PublicKeyHash local_id) { + return std::make_unique(std::move(opts), nodes, local_id); +} + +ValidatorSessionDescriptionImpl::MemPool::MemPool(size_t chunk_size) : chunk_size_(chunk_size) { +} + +ValidatorSessionDescriptionImpl::MemPool::~MemPool() { + for (auto &v : data_) { + delete[] v; + } +} + +void *ValidatorSessionDescriptionImpl::MemPool::alloc(size_t size, size_t align) { + CHECK(align && !(align & (align - 1))); // align should be a power of 2 + CHECK(size + align <= chunk_size_); auto get_padding = [&](const uint8_t* ptr) { return (-(size_t)ptr) & (align - 1); }; - if (temp) { - pdata_temp_ptr_ += get_padding(pdata_temp_ + pdata_temp_ptr_); - auto s = pdata_temp_ptr_; - pdata_temp_ptr_ += size; - CHECK(s + size <= pdata_temp_size_); - return static_cast(pdata_temp_ + s); - } else { - while (true) { - size_t idx = pdata_perm_ptr_ / pdata_perm_size_; - if (idx < pdata_perm_.size()) { - auto ptr = pdata_perm_[idx] + (pdata_perm_ptr_ % pdata_perm_size_); - pdata_perm_ptr_ += get_padding(ptr); - ptr += get_padding(ptr); - pdata_perm_ptr_ += size; - if (pdata_perm_ptr_ <= pdata_perm_.size() * pdata_perm_size_) { - return static_cast(ptr); - } + while (true) { + size_t idx = ptr_ / chunk_size_; + if (idx < data_.size()) { + auto ptr = data_[idx] + (ptr_ % chunk_size_); + ptr_ += get_padding(ptr); + ptr += get_padding(ptr); + ptr_ += size; + if (ptr_ <= data_.size() * chunk_size_) { + return static_cast(ptr); + } else { + ptr_ = data_.size() * chunk_size_; } - pdata_perm_.push_back(new td::uint8[pdata_perm_size_]); } + data_.push_back(new td::uint8[chunk_size_]); } } -bool ValidatorSessionDescriptionImpl::is_persistent(const void *ptr) const { +void ValidatorSessionDescriptionImpl::MemPool::clear() { + while (data_.size() > 1) { + delete[] data_.back(); + data_.pop_back(); + } + ptr_ = 0; +} + +bool ValidatorSessionDescriptionImpl::MemPool::contains(const void* ptr) const { if (ptr == nullptr) { return true; } - for (auto &v : pdata_perm_) { - if (ptr >= v && ptr <= v + pdata_perm_size_) { + for (auto &v : data_) { + if (ptr >= v && ptr <= v + chunk_size_) { return true; } } return false; } -std::unique_ptr ValidatorSessionDescription::create( - ValidatorSessionOptions opts, std::vector &nodes, PublicKeyHash local_id) { - return std::make_unique(std::move(opts), nodes, local_id); -} - } // namespace validatorsession } // namespace ton diff --git a/validator-session/validator-session-description.hpp b/validator-session/validator-session-description.hpp index d65369ace..0c662b7a2 100644 --- a/validator-session/validator-session-description.hpp +++ b/validator-session/validator-session-description.hpp @@ -50,20 +50,30 @@ class ValidatorSessionDescriptionImpl : public ValidatorSessionDescription { td::uint32 self_idx_; static constexpr td::uint32 cache_size = (1 << 20); + static constexpr size_t mem_chunk_size_perm = (1 << 27); + static constexpr size_t mem_chunk_size_temp = (1 << 27); struct Cached { const RootObject *ptr; }; std::array, cache_size> cache_; - //std::array, cache_size> temp_cache_; - td::uint8 *pdata_temp_; - size_t pdata_temp_ptr_; - size_t pdata_temp_size_; + class MemPool { + public: + explicit MemPool(size_t chunk_size); + ~MemPool(); + void *alloc(size_t size, size_t align); + void clear(); + bool contains(const void* ptr) const; + + private: + size_t chunk_size_; + std::vector data_; + size_t ptr_ = 0; + }; + MemPool mem_perm_ = MemPool(mem_chunk_size_perm); + MemPool mem_temp_ = MemPool(mem_chunk_size_temp); - size_t pdata_perm_size_; - std::vector pdata_perm_; - size_t pdata_perm_ptr_; std::atomic reuse_{0}; public: @@ -116,7 +126,7 @@ class ValidatorSessionDescriptionImpl : public ValidatorSessionDescription { void update_hash(const RootObject *obj, HashType hash) override; void *alloc(size_t size, size_t align, bool temp) override; void clear_temp_memory() override { - pdata_temp_ptr_ = 0; + mem_temp_.clear(); } bool is_persistent(const void *ptr) const override; HashType compute_hash(td::Slice data) const override; @@ -152,12 +162,6 @@ class ValidatorSessionDescriptionImpl : public ValidatorSessionDescription { const ValidatorSessionOptions &opts() const override { return opts_; } - ~ValidatorSessionDescriptionImpl() { - delete[] pdata_temp_; - for (auto &x : pdata_perm_) { - delete[] x; - } - } }; } // namespace validatorsession diff --git a/validator-session/validator-session-round-attempt-state.cpp b/validator-session/validator-session-round-attempt-state.cpp new file mode 100644 index 000000000..33c3d8047 --- /dev/null +++ b/validator-session/validator-session-round-attempt-state.cpp @@ -0,0 +1,486 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . + + Copyright 2017-2020 Telegram Systems LLP +*/ +#include "validator-session-state.h" +#include "td/utils/Random.h" +#include "auto/tl/ton_api.hpp" + +#include + +namespace ton { + +namespace validatorsession { + +static const SessionVoteCandidate* get_candidate(const VoteVector* vec, ValidatorSessionCandidateId id) { + if (!vec) { + return nullptr; + } + auto size = vec->size(); + auto v = vec->data(); + for (td::uint32 i = 0; i < size; i++) { + if (v[i]->get_id() == id) { + return v[i]; + } + } + return nullptr; +} + +// +// +// SessionBlockCandidateSignature +// +// + +const SessionBlockCandidateSignature* SessionBlockCandidateSignature::merge(ValidatorSessionDescription& desc, + const SessionBlockCandidateSignature* l, + const SessionBlockCandidateSignature* r) { + if (!l) { + return r; + } + if (!r) { + return l; + } + if (l == r) { + return l; + } + if (l->as_slice() < r->as_slice()) { + return l; + } else { + return r; + } +} + +// +// +// SessionBlockCandidate +// +// + +bool SessionBlockCandidate::check_block_is_approved(ValidatorSessionDescription& desc) const { + ValidatorWeight w = 0; + for (td::uint32 i = 0; i < desc.get_total_nodes(); i++) { + if (approved_by_->at(i)) { + w += desc.get_node_weight(i); + if (w >= desc.get_cutoff_weight()) { + return true; + } + } + } + return false; +} + +const SessionBlockCandidate* SessionBlockCandidate::merge(ValidatorSessionDescription& desc, + const SessionBlockCandidate* l, + const SessionBlockCandidate* r) { + if (!l) { + return r; + } + if (!r) { + return l; + } + if (l == r) { + return l; + } + CHECK(l->get_id() == r->get_id()); + auto v = SessionBlockCandidateSignatureVector::merge( + desc, l->approved_by_, r->approved_by_, + [&](const SessionBlockCandidateSignature* l, const SessionBlockCandidateSignature* r) { + return SessionBlockCandidateSignature::merge(desc, l, r); + }); + return SessionBlockCandidate::create(desc, l->block_, std::move(v)); +} + +// +// +// SessionVoteCandidate +// +// + +bool SessionVoteCandidate::check_block_is_voted(ValidatorSessionDescription& desc) const { + ValidatorWeight w = 0; + for (td::uint32 i = 0; i < desc.get_total_nodes(); i++) { + if (voted_by_->at(i)) { + w += desc.get_node_weight(i); + if (w >= desc.get_cutoff_weight()) { + return true; + } + } + } + return false; +} + +const SessionVoteCandidate* SessionVoteCandidate::merge(ValidatorSessionDescription& desc, + const SessionVoteCandidate* l, const SessionVoteCandidate* r) { + if (!l) { + return r; + } + if (!r) { + return l; + } + if (l == r) { + return l; + } + CHECK(l->get_id() == r->get_id()); + auto v = CntVector::merge(desc, l->voted_by_, r->voted_by_); + return SessionVoteCandidate::create(desc, l->block_, std::move(v)); +} + +// +// +// ATTEMPT STATE +// +// + +const ValidatorSessionRoundAttemptState* ValidatorSessionRoundAttemptState::merge( + ValidatorSessionDescription& desc, const ValidatorSessionRoundAttemptState* left, + const ValidatorSessionRoundAttemptState* right) { + if (!left) { + return right; + } + if (!right) { + return left; + } + if (left == right) { + return left; + } + CHECK(left->seqno_ == right->seqno_); + + const SentBlock* vote_for = nullptr; + bool vote_for_inited = false; + if (!left->vote_for_inited_) { + vote_for = right->vote_for_; + vote_for_inited = right->vote_for_inited_; + } else if (!right->vote_for_inited_) { + vote_for = left->vote_for_; + vote_for_inited = left->vote_for_inited_; + } else if (left->vote_for_ == right->vote_for_) { + vote_for_inited = true; + vote_for = left->vote_for_; + } else { + auto l = SentBlock::get_block_id(left->vote_for_); + auto r = SentBlock::get_block_id(right->vote_for_); + + vote_for_inited = true; + if (l < r) { + vote_for = left->vote_for_; + } else { + vote_for = right->vote_for_; + } + } + + auto precommitted = CntVector::merge(desc, left->precommitted_, right->precommitted_); + auto votes = VoteVector::merge(desc, left->votes_, right->votes_, + [&](const SessionVoteCandidate* l, const SessionVoteCandidate* r) { + return SessionVoteCandidate::merge(desc, l, r); + }); + + return ValidatorSessionRoundAttemptState::create(desc, left->seqno_, std::move(votes), std::move(precommitted), + vote_for, vote_for_inited); +} + +const ValidatorSessionRoundAttemptState* ValidatorSessionRoundAttemptState::action( + ValidatorSessionDescription& desc, const ValidatorSessionRoundAttemptState* state, td::uint32 src_idx, + td::uint32 att, const ton_api::validatorSession_message_voteFor& act, const ValidatorSessionRoundState* round) { + if (state->vote_for_inited_) { + VLOG(VALIDATOR_SESSION_WARNING) << "[validator session][node " << desc.get_source_id(src_idx) << "][" << act + << "]: invalid message: duplicate VOTEFOR"; + return state; + } + if (src_idx != desc.get_vote_for_author(att)) { + VLOG(VALIDATOR_SESSION_WARNING) << "[validator session][node " << desc.get_source_id(src_idx) << "][" << act + << "]: invalid message: bad VOTEFOR author"; + return state; + } + if (round->get_first_attempt(src_idx) == 0 && desc.opts().max_round_attempts > 0) { + VLOG(VALIDATOR_SESSION_WARNING) << "[validator session][node " << desc.get_source_id(src_idx) << "][" << act + << "]: invalid message: too early for VOTEFOR"; + return state; + } + if (round->get_first_attempt(src_idx) + desc.opts().max_round_attempts > att && desc.opts().max_round_attempts > 0) { + VLOG(VALIDATOR_SESSION_WARNING) << "[validator session][node " << desc.get_source_id(src_idx) << "][" << act + << "]: invalid message: too early for VOTEFOR"; + return state; + } + + auto x = round->get_block(act.candidate_); + + if (!x) { + VLOG(VALIDATOR_SESSION_WARNING) << "[validator session][node " << desc.get_source_id(src_idx) << "][" << act + << "]: invalid message: VOTEFOR for not submitted block"; + return state; + } + if (!x->check_block_is_approved(desc)) { + VLOG(VALIDATOR_SESSION_WARNING) << "[validator session][node " << desc.get_source_id(src_idx) << "][" << act + << "]: invalid message: VOTEFOR for not approved block"; + return state; + } + + return ValidatorSessionRoundAttemptState::create(desc, state->seqno_, state->votes_, state->precommitted_, + x->get_block(), true); +} + +const ValidatorSessionRoundAttemptState* ValidatorSessionRoundAttemptState::action( + ValidatorSessionDescription& desc, const ValidatorSessionRoundAttemptState* state, td::uint32 src_idx, + td::uint32 att, const ton_api::validatorSession_message_vote& act, const ValidatorSessionRoundState* round) { + bool made; + return make_one(desc, state, src_idx, att, round, &act, made); +} + +const ValidatorSessionRoundAttemptState* ValidatorSessionRoundAttemptState::action( + ValidatorSessionDescription& desc, const ValidatorSessionRoundAttemptState* state, td::uint32 src_idx, + td::uint32 att, const ton_api::validatorSession_message_precommit& act, const ValidatorSessionRoundState* round) { + bool made; + return make_one(desc, state, src_idx, att, round, &act, made); +} + +const ValidatorSessionRoundAttemptState* ValidatorSessionRoundAttemptState::action( + ValidatorSessionDescription& desc, const ValidatorSessionRoundAttemptState* state, td::uint32 src_idx, + td::uint32 att, const ton_api::validatorSession_message_empty& act, const ValidatorSessionRoundState* round) { + bool made; + return make_one(desc, state, src_idx, att, round, &act, made); +} + +const ValidatorSessionRoundAttemptState* ValidatorSessionRoundAttemptState::try_vote( + ValidatorSessionDescription& desc, const ValidatorSessionRoundAttemptState* state, td::uint32 src_idx, + td::uint32 att, const ValidatorSessionRoundState* round, const ton_api::validatorSession_round_Message* act, + bool& made) { + made = false; + if (state->check_vote_received_from(src_idx)) { + return state; + } + auto found = false; + auto block = round->choose_block_to_vote(desc, src_idx, att, state->vote_for_, state->vote_for_inited_, found); + if (!found) { + return state; + } + auto block_id = SentBlock::get_block_id(block); + made = true; + + if (act) { + if (act->get_id() != ton_api::validatorSession_message_vote::ID) { + VLOG(VALIDATOR_SESSION_WARNING) << "[validator session][node " << desc.get_source_id(src_idx) << "][" << act + << "]: expected VOTE(" << block_id << ")"; + } else { + auto x = static_cast(act); + if (x->candidate_ != block_id) { + VLOG(VALIDATOR_SESSION_WARNING) << "[validator session][node " << desc.get_source_id(src_idx) << "][" << act + << "]: expected VOTE(" << block_id << ")"; + } + } + } else { + VLOG(VALIDATOR_SESSION_WARNING) << "[validator session][node " << desc.get_source_id(src_idx) + << "]: making implicit VOTE(" << block_id << ")"; + } + + auto candidate = get_candidate(state->votes_, block_id); + if (!candidate) { + candidate = SessionVoteCandidate::create(desc, block); + } + candidate = SessionVoteCandidate::push(desc, candidate, src_idx); + auto v = VoteVector::push(desc, state->votes_, candidate); + return ValidatorSessionRoundAttemptState::create(desc, state->seqno_, std::move(v), state->precommitted_, + state->vote_for_, state->vote_for_inited_); +} + +const ValidatorSessionRoundAttemptState* ValidatorSessionRoundAttemptState::try_precommit( + ValidatorSessionDescription& desc, const ValidatorSessionRoundAttemptState* state, td::uint32 src_idx, + td::uint32 att, const ValidatorSessionRoundState* round, const ton_api::validatorSession_round_Message* act, + bool& made) { + made = false; + if (state->check_precommit_received_from(src_idx)) { + return state; + } + bool found; + auto block = state->get_voted_block(desc, found); + if (!found) { + return state; + } + made = true; + auto block_id = SentBlock::get_block_id(block); + + if (act) { + if (act->get_id() != ton_api::validatorSession_message_precommit::ID) { + VLOG(VALIDATOR_SESSION_WARNING) << "[validator session][node " << desc.get_source_id(src_idx) << "][" << act + << "]: expected PRECOMMIT(" << block_id << ")"; + } else { + auto x = static_cast(act); + if (x->candidate_ != block_id) { + VLOG(VALIDATOR_SESSION_WARNING) << "[validator session][node " << desc.get_source_id(src_idx) << "][" << act + << "]: expected PRECOMMIT(" << block_id << ")"; + } + } + } else { + VLOG(VALIDATOR_SESSION_WARNING) << "[validator session][node " << desc.get_source_id(src_idx) + << "]: making implicit PRECOMMIT(" << block_id << ")"; + } + + auto v = CntVector::change(desc, state->precommitted_, src_idx, true); + return ValidatorSessionRoundAttemptState::create(desc, state->seqno_, state->votes_, std::move(v), state->vote_for_, + state->vote_for_inited_); +} + +const ValidatorSessionRoundAttemptState* ValidatorSessionRoundAttemptState::make_one( + ValidatorSessionDescription& desc, const ValidatorSessionRoundAttemptState* state, td::uint32 src_idx, + td::uint32 att, const ValidatorSessionRoundState* round, const ton_api::validatorSession_round_Message* act, + bool& made) { + made = false; + state = try_vote(desc, state, src_idx, att, round, act, made); + if (made) { + return state; + } + state = try_precommit(desc, state, src_idx, att, round, act, made); + if (made) { + return state; + } + if (act && act->get_id() != ton_api::validatorSession_message_empty::ID) { + VLOG(VALIDATOR_SESSION_WARNING) << "[validator session][node " << desc.get_source_id(src_idx) << "][" << act + << "]: invalid message: expected EMPTY"; + } + return state; +} + +const ValidatorSessionRoundAttemptState* ValidatorSessionRoundAttemptState::action( + ValidatorSessionDescription& desc, const ValidatorSessionRoundAttemptState* state, td::uint32 src_idx, + td::uint32 att, const ton_api::validatorSession_round_Message* act, const ValidatorSessionRoundState* round) { + ton_api::downcast_call(*const_cast(act), + [&](auto& obj) { state = action(desc, state, src_idx, att, obj, round); }); + return state; +} + +bool ValidatorSessionRoundAttemptState::check_vote_received_from(td::uint32 src_idx) const { + if (!votes_) { + return false; + } + auto size = votes_->size(); + auto v = votes_->data(); + for (td::uint32 i = 0; i < size; i++) { + if (v[i]->check_block_is_voted_by(src_idx)) { + return true; + } + } + return false; +} + +bool ValidatorSessionRoundAttemptState::check_precommit_received_from(td::uint32 src_idx) const { + return precommitted_->at(src_idx); +} + +const SentBlock* ValidatorSessionRoundAttemptState::get_voted_block(ValidatorSessionDescription& desc, bool& f) const { + f = false; + if (!votes_) { + return nullptr; + } + + auto size = votes_->size(); + auto v = votes_->data(); + for (td::uint32 i = 0; i < size; i++) { + if (v[i]->check_block_is_voted(desc)) { + f = true; + return v[i]->get_block(); + } + } + return nullptr; +} + +bool ValidatorSessionRoundAttemptState::check_attempt_is_precommitted(ValidatorSessionDescription& desc) const { + ValidatorWeight weight = 0; + for (td::uint32 i = 0; i < desc.get_total_nodes(); i++) { + if (precommitted_->at(i)) { + weight += desc.get_node_weight(i); + if (weight >= desc.get_cutoff_weight()) { + return true; + } + } + } + return false; +} + +tl_object_ptr ValidatorSessionRoundAttemptState::create_action( + ValidatorSessionDescription& desc, const ValidatorSessionRoundState* round, td::uint32 src_idx, + td::uint32 att) const { + if (!check_vote_received_from(src_idx)) { + auto found = false; + auto B = round->choose_block_to_vote(desc, src_idx, att, vote_for_, vote_for_inited_, found); + if (found) { + auto block_id = SentBlock::get_block_id(B); + return create_tl_object(round->get_seqno(), seqno_, block_id); + } + } + if (!check_precommit_received_from(src_idx)) { + bool f = false; + auto B = get_voted_block(desc, f); + + if (f) { + auto block_id = SentBlock::get_block_id(B); + + return create_tl_object(round->get_seqno(), seqno_, block_id); + } + } + + return create_tl_object(round->get_seqno(), seqno_); +} + +void ValidatorSessionRoundAttemptState::dump(ValidatorSessionDescription& desc, td::StringBuilder& sb) const { + sb << "attempt=" << seqno_ << "\n"; + sb << ">>>>\n"; + + if (vote_for_inited_) { + sb << "vote_for=" << (vote_for_ ? vote_for_->get_src_idx() : std::numeric_limits::max()) << "\n"; + } else { + sb << "vote_for=NONE\n"; + } + + if (votes_) { + auto s = votes_->size(); + sb << "votes: "; + std::vector R; + R.resize(desc.get_total_nodes(), -1); + for (td::uint32 i = 0; i < s; i++) { + const auto e = votes_->at(i); + const auto& x = e->get_voters_list(); + for (td::uint32 j = 0; j < desc.get_total_nodes(); j++) { + if (x->at(j)) { + R[j] = e->get_src_idx(); + } + } + } + for (td::uint32 i = 0; i < desc.get_total_nodes(); i++) { + sb << R[i] << " "; + } + sb << "\n"; + } else { + sb << "votes: EMPTY\n"; + } + + sb << "precommits: "; + for (td::uint32 i = 0; i < desc.get_total_nodes(); i++) { + const auto e = precommitted_->at(i); + if (e) { + sb << "+ "; + } else { + sb << "- "; + } + } + sb << "\n"; + sb << "<<<<\n"; +} + + +} // namespace validatorsession + +} // namespace ton diff --git a/validator-session/validator-session-round-attempt-state.h b/validator-session/validator-session-round-attempt-state.h new file mode 100644 index 000000000..a4a6b68bd --- /dev/null +++ b/validator-session/validator-session-round-attempt-state.h @@ -0,0 +1,620 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ + +#pragma once + +#include "td/utils/int_types.h" +#include "td/utils/buffer.h" +#include "adnl/utils.hpp" +#include "common/io.hpp" + +#include "persistent-vector.h" + +#include "validator-session-description.h" + +#include "validator-session-types.h" + +#include + + +namespace ton { + +namespace validatorsession { + +using HashType = ValidatorSessionDescription::HashType; + +struct SessionBlockCandidateSignature : public ValidatorSessionDescription::RootObject { + public: + static auto create_hash(ValidatorSessionDescription& desc, td::Slice data) { + auto obj = create_tl_object(desc.compute_hash(data)); + return desc.compute_hash(serialize_tl_object(obj, true).as_slice()); + } + + static bool compare(const RootObject* r, td::Slice data, HashType hash) { + if (!r || r->get_size() < sizeof(SessionBlockCandidateSignature)) { + return false; + } + auto R = static_cast(r); + return R->hash_ == hash && R->data_.ubegin() == data.ubegin() && R->data_.size() == data.size(); + } + + static auto lookup(ValidatorSessionDescription& desc, td::Slice data, HashType hash, bool temp) { + auto r = desc.get_by_hash(hash, temp); + if (compare(r, data, hash)) { + desc.on_reuse(); + return static_cast(r); + } + return static_cast(nullptr); + } + static SessionBlockCandidateSignature* create(ValidatorSessionDescription& desc, td::BufferSlice value) { + auto hash = create_hash(desc, value.as_slice()); + auto d = static_cast(desc.alloc(value.size(), 8, false)); + td::MutableSlice s{d, value.size()}; + s.copy_from(value.as_slice()); + return new (desc, true) SessionBlockCandidateSignature{desc, s, hash}; + } + static const SessionBlockCandidateSignature* move_to_persistent(ValidatorSessionDescription& desc, + const SessionBlockCandidateSignature* b) { + if (desc.is_persistent(b)) { + return b; + } + CHECK(desc.is_persistent(b->data_.ubegin())); + auto r = lookup(desc, b->data_, b->hash_, false); + if (r) { + return r; + } + return new (desc, false) SessionBlockCandidateSignature{desc, b->data_, b->hash_}; + } + static const SessionBlockCandidateSignature* merge(ValidatorSessionDescription& desc, + const SessionBlockCandidateSignature* l, + const SessionBlockCandidateSignature* r); + SessionBlockCandidateSignature(ValidatorSessionDescription& desc, td::Slice data, HashType hash) + : RootObject(sizeof(SessionBlockCandidateSignature)), data_{data}, hash_(std::move(hash)) { + desc.update_hash(this, hash_); + } + td::BufferSlice value() const { + return td::BufferSlice{data_}; + } + td::Slice as_slice() const { + return data_; + } + auto get_hash(ValidatorSessionDescription& desc) const { + return hash_; + } + + private: + const td::Slice data_; + const HashType hash_; +}; + +using SessionBlockCandidateSignatureVector = CntVector; + +class SentBlock : public ValidatorSessionDescription::RootObject { + public: + static HashType create_hash(ValidatorSessionDescription& desc, td::uint32 src_idx, ValidatorSessionRootHash root_hash, + ValidatorSessionFileHash file_hash, + ValidatorSessionCollatedDataFileHash collated_data_file_hash) { + auto obj = create_tl_object(src_idx, get_vs_hash(desc, root_hash), + get_vs_hash(desc, file_hash), + get_vs_hash(desc, collated_data_file_hash)); + return desc.compute_hash(serialize_tl_object(obj, true).as_slice()); + } + static bool compare(const RootObject* root_object, td::uint32 src_idx, const ValidatorSessionRootHash& root_hash, + const ValidatorSessionFileHash& file_hash, + const ValidatorSessionCollatedDataFileHash& collated_data_file_hash, HashType hash) { + if (!root_object || root_object->get_size() < sizeof(SentBlock)) { + return false; + } + auto obj = static_cast(root_object); + return obj->src_idx_ == src_idx && obj->root_hash_ == root_hash && obj->file_hash_ == file_hash && + obj->collated_data_file_hash_ == collated_data_file_hash && obj->hash_ == hash; + } + static auto lookup(ValidatorSessionDescription& desc, td::uint32 src_idx, const ValidatorSessionRootHash& root_hash, + const ValidatorSessionFileHash& file_hash, + const ValidatorSessionCollatedDataFileHash& collated_data_file_hash, HashType hash, bool temp) { + auto r = desc.get_by_hash(hash, temp); + if (compare(r, src_idx, root_hash, file_hash, collated_data_file_hash, hash)) { + desc.on_reuse(); + return static_cast(r); + } + return static_cast(nullptr); + } + static const SentBlock* create(ValidatorSessionDescription& desc, td::uint32 src_idx, + const ValidatorSessionRootHash& root_hash, const ValidatorSessionFileHash& file_hash, + const ValidatorSessionCollatedDataFileHash& collated_data_file_hash) { + auto hash = create_hash(desc, src_idx, root_hash, file_hash, collated_data_file_hash); + auto r = lookup(desc, src_idx, root_hash, file_hash, collated_data_file_hash, hash, true); + if (r) { + return r; + } + + auto candidate_id = desc.candidate_id(src_idx, root_hash, file_hash, collated_data_file_hash); + + return new (desc, true) SentBlock{desc, src_idx, root_hash, file_hash, collated_data_file_hash, candidate_id, hash}; + } + static const SentBlock* create(ValidatorSessionDescription& desc, const ValidatorSessionCandidateId& zero) { + CHECK(zero.is_zero()); + auto hash = create_hash(desc, 0, ValidatorSessionRootHash::zero(), ValidatorSessionFileHash::zero(), + ValidatorSessionCollatedDataFileHash::zero()); + + return new (desc, true) SentBlock{desc, + 0, + ValidatorSessionRootHash::zero(), + ValidatorSessionFileHash::zero(), + ValidatorSessionCollatedDataFileHash::zero(), + zero, + hash}; + } + static const SentBlock* move_to_persistent(ValidatorSessionDescription& desc, const SentBlock* b) { + if (desc.is_persistent(b)) { + return b; + } + auto r = lookup(desc, b->src_idx_, b->root_hash_, b->file_hash_, b->collated_data_file_hash_, b->hash_, false); + if (r) { + return r; + } + + return new (desc, false) SentBlock{ + desc, b->src_idx_, b->root_hash_, b->file_hash_, b->collated_data_file_hash_, b->candidate_id_, b->hash_}; + } + SentBlock(ValidatorSessionDescription& desc, td::uint32 src_idx, ValidatorSessionRootHash root_hash, + ValidatorSessionFileHash file_hash, ValidatorSessionCollatedDataFileHash collated_data_file_hash, + ValidatorSessionCandidateId candidate_id, HashType hash) + : RootObject(sizeof(SentBlock)) + , src_idx_(src_idx) + , root_hash_(std::move(root_hash)) + , file_hash_(std::move(file_hash)) + , collated_data_file_hash_(std::move(collated_data_file_hash)) + , candidate_id_(candidate_id) + , hash_(std::move(hash)) { + desc.update_hash(this, hash_); + } + auto get_src_idx() const { + return src_idx_; + } + auto get_root_hash() const { + return root_hash_; + } + auto get_file_hash() const { + return file_hash_; + } + auto get_collated_data_file_hash() const { + return collated_data_file_hash_; + } + static ValidatorSessionCandidateId get_block_id(const SentBlock* block) { + return block ? block->candidate_id_ : skip_round_candidate_id(); + } + HashType get_hash(ValidatorSessionDescription& desc) const { + return hash_; + } + bool operator<(const SentBlock& block) const { + if (src_idx_ < block.src_idx_) { + return true; + } + if (src_idx_ > block.src_idx_) { + return false; + } + if (candidate_id_ < block.candidate_id_) { + return true; + } + return false; + } + struct Compare { + bool operator()(const SentBlock* a, const SentBlock* b) const { + return *a < *b; + } + }; + + private: + const td::uint32 src_idx_; + const ValidatorSessionRootHash root_hash_; + const ValidatorSessionFileHash file_hash_; + const ValidatorSessionCollatedDataFileHash collated_data_file_hash_; + const ValidatorSessionCandidateId candidate_id_; + const HashType hash_; +}; + +class SessionBlockCandidate : public ValidatorSessionDescription::RootObject { + public: + static HashType create_hash(ValidatorSessionDescription& desc, HashType block, HashType approved) { + auto obj = create_tl_object(block, approved); + return desc.compute_hash(serialize_tl_object(obj, true).as_slice()); + } + static bool compare(const RootObject* r, const SentBlock* block, const SessionBlockCandidateSignatureVector* approved, + HashType hash) { + if (!r || r->get_size() < sizeof(SessionBlockCandidate)) { + return false; + } + auto R = static_cast(r); + return R->block_ == block && R->approved_by_ == approved && R->hash_ == hash; + } + static auto lookup(ValidatorSessionDescription& desc, const SentBlock* block, + const SessionBlockCandidateSignatureVector* approved, HashType hash, bool temp) { + auto r = desc.get_by_hash(hash, temp); + if (compare(r, block, approved, hash)) { + desc.on_reuse(); + return static_cast(r); + } + return static_cast(nullptr); + } + static const SessionBlockCandidate* create(ValidatorSessionDescription& desc, const SentBlock* block, + const SessionBlockCandidateSignatureVector* approved) { + auto hash = create_hash(desc, get_vs_hash(desc, block), get_vs_hash(desc, approved)); + + auto r = lookup(desc, block, approved, hash, true); + if (r) { + return r; + } + + return new (desc, true) SessionBlockCandidate(desc, block, approved, hash); + } + static const SessionBlockCandidate* create(ValidatorSessionDescription& desc, const SentBlock* block) { + std::vector v; + v.resize(desc.get_total_nodes(), nullptr); + auto vec = SessionBlockCandidateSignatureVector::create(desc, std::move(v)); + return create(desc, block, vec); + } + static const SessionBlockCandidate* move_to_persistent(ValidatorSessionDescription& desc, + const SessionBlockCandidate* b) { + if (desc.is_persistent(b)) { + return b; + } + auto block = SentBlock::move_to_persistent(desc, b->block_); + auto approved = SessionBlockCandidateSignatureVector::move_to_persistent(desc, b->approved_by_); + auto r = lookup(desc, block, approved, b->hash_, false); + if (r) { + return r; + } + + return new (desc, false) SessionBlockCandidate{desc, block, approved, b->hash_}; + } + SessionBlockCandidate(ValidatorSessionDescription& desc, const SentBlock* block, + const SessionBlockCandidateSignatureVector* approved, HashType hash) + : RootObject{sizeof(SessionBlockCandidate)}, block_(block), approved_by_(approved), hash_(hash) { + desc.update_hash(this, hash_); + } + static const SessionBlockCandidate* merge(ValidatorSessionDescription& desc, const SessionBlockCandidate* l, + const SessionBlockCandidate* r); + auto get_block() const { + return block_; + } + auto get_id() const { + return SentBlock::get_block_id(block_); + } + auto get_src_idx() const { + return block_ ? block_->get_src_idx() : std::numeric_limits::max(); + } + bool check_block_is_approved_by(td::uint32 src_idx) const { + return approved_by_->at(src_idx); + } + bool check_block_is_approved(ValidatorSessionDescription& desc) const; + auto get_hash(ValidatorSessionDescription& desc) const { + return hash_; + } + auto get_approvers_list() const { + return approved_by_; + } + static const SessionBlockCandidate* push(ValidatorSessionDescription& desc, const SessionBlockCandidate* state, + td::uint32 src_idx, const SessionBlockCandidateSignature* sig) { + CHECK(state); + if (state->approved_by_->at(src_idx)) { + return state; + } + return create(desc, state->block_, + SessionBlockCandidateSignatureVector::change(desc, state->approved_by_, src_idx, sig)); + } + class Compare { + public: + bool operator()(const SessionBlockCandidate* l, const SessionBlockCandidate* r) { + return l->get_id() < r->get_id(); + } + }; + + private: + const SentBlock* block_; + const SessionBlockCandidateSignatureVector* approved_by_; + const HashType hash_; +}; + +class SessionVoteCandidate : public ValidatorSessionDescription::RootObject { + public: + static HashType create_hash(ValidatorSessionDescription& desc, HashType block, HashType voted) { + auto obj = create_tl_object(block, voted); + return desc.compute_hash(serialize_tl_object(obj, true).as_slice()); + } + static bool compare(const RootObject* r, const SentBlock* block, const CntVector* voted, HashType hash) { + if (!r || r->get_size() < sizeof(SessionVoteCandidate)) { + return false; + } + auto R = static_cast(r); + return R->block_ == block && R->voted_by_ == voted && R->hash_ == hash; + } + static auto lookup(ValidatorSessionDescription& desc, const SentBlock* block, const CntVector* voted, + HashType hash, bool temp) { + auto r = desc.get_by_hash(hash, temp); + if (compare(r, block, voted, hash)) { + desc.on_reuse(); + return static_cast(r); + } + return static_cast(nullptr); + } + static const SessionVoteCandidate* create(ValidatorSessionDescription& desc, const SentBlock* block, + const CntVector* voted) { + auto hash = create_hash(desc, get_vs_hash(desc, block), get_vs_hash(desc, voted)); + + auto r = lookup(desc, block, voted, hash, true); + if (r) { + return r; + } + + return new (desc, true) SessionVoteCandidate(desc, block, voted, hash); + } + static const SessionVoteCandidate* create(ValidatorSessionDescription& desc, const SentBlock* block) { + std::vector v; + v.resize(desc.get_total_nodes(), false); + auto vec = CntVector::create(desc, std::move(v)); + return create(desc, block, vec); + } + static const SessionVoteCandidate* move_to_persistent(ValidatorSessionDescription& desc, + const SessionVoteCandidate* b) { + if (desc.is_persistent(b)) { + return b; + } + auto block = SentBlock::move_to_persistent(desc, b->block_); + auto voted = CntVector::move_to_persistent(desc, b->voted_by_); + auto r = lookup(desc, block, voted, b->hash_, false); + if (r) { + return r; + } + + return new (desc, false) SessionVoteCandidate{desc, block, voted, b->hash_}; + } + SessionVoteCandidate(ValidatorSessionDescription& desc, const SentBlock* block, const CntVector* voted, + HashType hash) + : RootObject{sizeof(SessionVoteCandidate)}, block_(block), voted_by_(voted), hash_(hash) { + desc.update_hash(this, hash_); + } + static const SessionVoteCandidate* merge(ValidatorSessionDescription& desc, const SessionVoteCandidate* l, + const SessionVoteCandidate* r); + auto get_block() const { + return block_; + } + auto get_id() const { + return SentBlock::get_block_id(block_); + } + auto get_src_idx() const { + return block_ ? block_->get_src_idx() : std::numeric_limits::max(); + } + bool check_block_is_voted_by(td::uint32 src_idx) const { + return voted_by_->at(src_idx); + } + bool check_block_is_voted(ValidatorSessionDescription& desc) const; + auto get_hash(ValidatorSessionDescription& desc) const { + return hash_; + } + auto get_voters_list() const { + return voted_by_; + } + static const SessionVoteCandidate* push(ValidatorSessionDescription& desc, const SessionVoteCandidate* state, + td::uint32 src_idx) { + CHECK(state); + if (state->voted_by_->at(src_idx)) { + return state; + } + return create(desc, state->block_, CntVector::change(desc, state->voted_by_, src_idx, true)); + } + class Compare { + public: + bool operator()(const SessionVoteCandidate* l, const SessionVoteCandidate* r) { + return l->get_id() < r->get_id(); + } + }; + + private: + const SentBlock* block_; + const CntVector* voted_by_; + const HashType hash_; +}; + +using VoteVector = CntSortedVector; +using ApproveVector = CntSortedVector; +class ValidatorSessionRoundState; + +class ValidatorSessionRoundAttemptState : public ValidatorSessionDescription::RootObject { + public: + static HashType create_hash(ValidatorSessionDescription& desc, td::uint32 seqno, HashType votes, + HashType precommitted, bool vote_for_inited, HashType vote_for) { + auto obj = create_tl_object(seqno, votes, precommitted, + vote_for_inited, vote_for); + return desc.compute_hash(serialize_tl_object(obj, true).as_slice()); + } + static bool compare(const RootObject* r, td::uint32 seqno, const VoteVector* votes, + const CntVector* precommitted, const SentBlock* vote_for, bool vote_for_inited, + HashType hash) { + if (!r || r->get_size() < sizeof(ValidatorSessionRoundAttemptState)) { + return false; + } + auto R = static_cast(r); + return R->seqno_ == seqno && R->votes_ == votes && R->precommitted_ == precommitted && R->vote_for_ == vote_for && + R->vote_for_inited_ == vote_for_inited && R->hash_ == hash; + } + static auto lookup(ValidatorSessionDescription& desc, td::uint32 seqno, const VoteVector* votes, + const CntVector* precommitted, const SentBlock* vote_for, bool vote_for_inited, + HashType hash, bool temp) { + auto r = desc.get_by_hash(hash, temp); + if (compare(r, seqno, votes, precommitted, vote_for, vote_for_inited, hash)) { + desc.on_reuse(); + return static_cast(r); + } + return static_cast(nullptr); + } + static const ValidatorSessionRoundAttemptState* move_to_persistent(ValidatorSessionDescription& desc, + const ValidatorSessionRoundAttemptState* b) { + if (desc.is_persistent(b)) { + return b; + } + auto votes = VoteVector::move_to_persistent(desc, b->votes_); + auto precommitted = CntVector::move_to_persistent(desc, b->precommitted_); + auto vote_for = SentBlock::move_to_persistent(desc, b->vote_for_); + + auto r = lookup(desc, b->seqno_, votes, precommitted, vote_for, b->vote_for_inited_, b->hash_, false); + if (r) { + return r; + } + + return new (desc, false) ValidatorSessionRoundAttemptState{desc, b->seqno_, votes, precommitted, + vote_for, b->vote_for_inited_, b->hash_}; + } + + static const ValidatorSessionRoundAttemptState* create(ValidatorSessionDescription& desc, td::uint32 seqno, + const VoteVector* votes, const CntVector* precommitted, + const SentBlock* vote_for, bool vote_for_inited) { + auto hash = create_hash(desc, seqno, get_vs_hash(desc, votes), get_vs_hash(desc, precommitted), + get_vs_hash(desc, vote_for), vote_for_inited); + + auto r = lookup(desc, seqno, votes, precommitted, vote_for, vote_for_inited, hash, true); + if (r) { + return r; + } + + return new (desc, true) + ValidatorSessionRoundAttemptState(desc, seqno, votes, precommitted, vote_for, vote_for_inited, hash); + } + static const ValidatorSessionRoundAttemptState* create(ValidatorSessionDescription& desc, td::uint32 seqno) { + std::vector x; + x.resize(desc.get_total_nodes(), false); + auto p = CntVector::create(desc, std::move(x)); + + return create(desc, seqno, nullptr, p, nullptr, false); + } + + ValidatorSessionRoundAttemptState(ValidatorSessionDescription& desc, td::uint32 seqno, const VoteVector* votes, + const CntVector* precommitted, const SentBlock* vote_for, + bool vote_for_inited, HashType hash) + : RootObject{sizeof(ValidatorSessionRoundAttemptState)} + , seqno_(seqno) + , votes_(votes) + , precommitted_(precommitted) + , vote_for_(vote_for) + , vote_for_inited_(vote_for_inited) + , hash_(std::move(hash)) { + desc.update_hash(this, hash_); + } + auto get_hash(ValidatorSessionDescription& desc) const { + return hash_; + } + auto get_seqno() const { + return seqno_; + } + auto get_votes() const { + return votes_; + } + auto get_precommits() const { + return precommitted_; + } + const SentBlock* get_voted_block(ValidatorSessionDescription& desc, bool& f) const; + const SentBlock* get_vote_for_block(ValidatorSessionDescription& desc, bool& f) const { + f = vote_for_inited_; + return vote_for_; + } + bool check_attempt_is_precommitted(ValidatorSessionDescription& desc) const; + bool check_vote_received_from(td::uint32 src_idx) const; + bool check_precommit_received_from(td::uint32 src_idx) const; + + bool operator<(const ValidatorSessionRoundAttemptState& right) const { + return seqno_ < right.seqno_; + } + struct Compare { + bool operator()(const ValidatorSessionRoundAttemptState* a, const ValidatorSessionRoundAttemptState* b) const { + return *a < *b; + } + }; + + static const ValidatorSessionRoundAttemptState* merge(ValidatorSessionDescription& desc, + const ValidatorSessionRoundAttemptState* left, + const ValidatorSessionRoundAttemptState* right); + static const ValidatorSessionRoundAttemptState* action(ValidatorSessionDescription& desc, + const ValidatorSessionRoundAttemptState* state, + td::uint32 src_idx, td::uint32 att, + const ton_api::validatorSession_message_voteFor& act, + const ValidatorSessionRoundState* round); + static const ValidatorSessionRoundAttemptState* action(ValidatorSessionDescription& desc, + const ValidatorSessionRoundAttemptState* state, + td::uint32 src_idx, td::uint32 att, + const ton_api::validatorSession_message_vote& act, + const ValidatorSessionRoundState* round); + static const ValidatorSessionRoundAttemptState* action(ValidatorSessionDescription& desc, + const ValidatorSessionRoundAttemptState* state, + td::uint32 src_idx, td::uint32 att, + const ton_api::validatorSession_message_precommit& act, + const ValidatorSessionRoundState* round); + static const ValidatorSessionRoundAttemptState* action(ValidatorSessionDescription& desc, + const ValidatorSessionRoundAttemptState* state, + td::uint32 src_idx, td::uint32 att, + const ton_api::validatorSession_message_empty& act, + const ValidatorSessionRoundState* round); + static const ValidatorSessionRoundAttemptState* action(ValidatorSessionDescription& desc, + const ValidatorSessionRoundAttemptState* state, + td::uint32 src_idx, td::uint32 att, + const ton_api::validatorSession_round_Message* action, + const ValidatorSessionRoundState* round); + template + static const ValidatorSessionRoundAttemptState* action(ValidatorSessionDescription& desc, + const ValidatorSessionRoundAttemptState* state, + td::uint32 src_idx, td::uint32 att, const T& action, + const ValidatorSessionRoundState* round) { + UNREACHABLE(); + } + static const ValidatorSessionRoundAttemptState* try_vote(ValidatorSessionDescription& desc, + const ValidatorSessionRoundAttemptState* state, + td::uint32 src_idx, td::uint32 att, + const ValidatorSessionRoundState* round, + const ton_api::validatorSession_round_Message* cmp, + bool& made); + static const ValidatorSessionRoundAttemptState* try_precommit(ValidatorSessionDescription& desc, + const ValidatorSessionRoundAttemptState* state, + td::uint32 src_idx, td::uint32 att, + const ValidatorSessionRoundState* round, + const ton_api::validatorSession_round_Message* cmp, + bool& made); + static const ValidatorSessionRoundAttemptState* make_one(ValidatorSessionDescription& desc, + const ValidatorSessionRoundAttemptState* state, + td::uint32 src_idx, td::uint32 att, + const ValidatorSessionRoundState* round, + const ton_api::validatorSession_round_Message* cmp, + bool& made); + tl_object_ptr create_action(ValidatorSessionDescription& desc, + const ValidatorSessionRoundState* round, + td::uint32 src_idx, td::uint32 att) const; + void dump(ValidatorSessionDescription& desc, td::StringBuilder& sb) const; + + private: + const td::uint32 seqno_; + const VoteVector* votes_; + const CntVector* precommitted_; + const SentBlock* vote_for_; + const bool vote_for_inited_; + const HashType hash_; +}; + +using AttemptVector = + CntSortedVector; + + +} // namespace validatorsession + +} // namespace ton diff --git a/validator-session/validator-session-state.cpp b/validator-session/validator-session-state.cpp index 62a054d54..97b918954 100644 --- a/validator-session/validator-session-state.cpp +++ b/validator-session/validator-session-state.cpp @@ -67,14 +67,6 @@ namespace ton { namespace validatorsession { -static td::uint32 get_round_id(const ton_api::validatorSession_round_Message* message) { - td::uint32 round = 0; - bool is_called = ton_api::downcast_call(*const_cast(message), - [&](auto& obj) { round = obj.round_; }); - CHECK(is_called); - return round; -} - static const ValidatorSessionRoundAttemptState* get_attempt(const AttemptVector* vec, td::uint32 seqno) { if (!vec) { return nullptr; @@ -108,118 +100,12 @@ static const SessionBlockCandidate* get_candidate(const ApproveVector* vec, Vali return nullptr; } -static const SessionVoteCandidate* get_candidate(const VoteVector* vec, ValidatorSessionCandidateId id) { - if (!vec) { - return nullptr; - } - auto size = vec->size(); - auto v = vec->data(); - for (td::uint32 i = 0; i < size; i++) { - if (v[i]->get_id() == id) { - return v[i]; - } - } - return nullptr; -} - -// -// -// SessionBlockCandidateSignature -// -// - -const SessionBlockCandidateSignature* SessionBlockCandidateSignature::merge(ValidatorSessionDescription& desc, - const SessionBlockCandidateSignature* l, - const SessionBlockCandidateSignature* r) { - if (!l) { - return r; - } - if (!r) { - return l; - } - if (l == r) { - return l; - } - if (l->as_slice() < r->as_slice()) { - return l; - } else { - return r; - } -} - -// -// -// SessionBlockCandidate -// -// - -bool SessionBlockCandidate::check_block_is_approved(ValidatorSessionDescription& desc) const { - ValidatorWeight w = 0; - for (td::uint32 i = 0; i < desc.get_total_nodes(); i++) { - if (approved_by_->at(i)) { - w += desc.get_node_weight(i); - if (w >= desc.get_cutoff_weight()) { - return true; - } - } - } - return false; -} - -const SessionBlockCandidate* SessionBlockCandidate::merge(ValidatorSessionDescription& desc, - const SessionBlockCandidate* l, - const SessionBlockCandidate* r) { - if (!l) { - return r; - } - if (!r) { - return l; - } - if (l == r) { - return l; - } - CHECK(l->get_id() == r->get_id()); - auto v = SessionBlockCandidateSignatureVector::merge( - desc, l->approved_by_, r->approved_by_, - [&](const SessionBlockCandidateSignature* l, const SessionBlockCandidateSignature* r) { - return SessionBlockCandidateSignature::merge(desc, l, r); - }); - return SessionBlockCandidate::create(desc, l->block_, std::move(v)); -} - -// -// -// SessionVoteCandidate -// -// - -bool SessionVoteCandidate::check_block_is_voted(ValidatorSessionDescription& desc) const { - ValidatorWeight w = 0; - for (td::uint32 i = 0; i < desc.get_total_nodes(); i++) { - if (voted_by_->at(i)) { - w += desc.get_node_weight(i); - if (w >= desc.get_cutoff_weight()) { - return true; - } - } - } - return false; -} - -const SessionVoteCandidate* SessionVoteCandidate::merge(ValidatorSessionDescription& desc, - const SessionVoteCandidate* l, const SessionVoteCandidate* r) { - if (!l) { - return r; - } - if (!r) { - return l; - } - if (l == r) { - return l; - } - CHECK(l->get_id() == r->get_id()); - auto v = CntVector::merge(desc, l->voted_by_, r->voted_by_); - return SessionVoteCandidate::create(desc, l->block_, std::move(v)); +static td::uint32 get_round_id(const ton_api::validatorSession_round_Message* message) { + td::uint32 round = 0; + bool is_called = ton_api::downcast_call(*const_cast(message), + [&](auto& obj) { round = obj.round_; }); + CHECK(is_called); + return round; } // @@ -374,346 +260,6 @@ const ValidatorSessionOldRoundState* ValidatorSessionOldRoundState::action( return state; } -// -// -// ATTEMPT STATE -// -// - -const ValidatorSessionRoundAttemptState* ValidatorSessionRoundAttemptState::merge( - ValidatorSessionDescription& desc, const ValidatorSessionRoundAttemptState* left, - const ValidatorSessionRoundAttemptState* right) { - if (!left) { - return right; - } - if (!right) { - return left; - } - if (left == right) { - return left; - } - CHECK(left->seqno_ == right->seqno_); - - const SentBlock* vote_for = nullptr; - bool vote_for_inited = false; - if (!left->vote_for_inited_) { - vote_for = right->vote_for_; - vote_for_inited = right->vote_for_inited_; - } else if (!right->vote_for_inited_) { - vote_for = left->vote_for_; - vote_for_inited = left->vote_for_inited_; - } else if (left->vote_for_ == right->vote_for_) { - vote_for_inited = true; - vote_for = left->vote_for_; - } else { - auto l = SentBlock::get_block_id(left->vote_for_); - auto r = SentBlock::get_block_id(right->vote_for_); - - vote_for_inited = true; - if (l < r) { - vote_for = left->vote_for_; - } else { - vote_for = right->vote_for_; - } - } - - auto precommitted = CntVector::merge(desc, left->precommitted_, right->precommitted_); - auto votes = VoteVector::merge(desc, left->votes_, right->votes_, - [&](const SessionVoteCandidate* l, const SessionVoteCandidate* r) { - return SessionVoteCandidate::merge(desc, l, r); - }); - - return ValidatorSessionRoundAttemptState::create(desc, left->seqno_, std::move(votes), std::move(precommitted), - vote_for, vote_for_inited); -} - -const ValidatorSessionRoundAttemptState* ValidatorSessionRoundAttemptState::action( - ValidatorSessionDescription& desc, const ValidatorSessionRoundAttemptState* state, td::uint32 src_idx, - td::uint32 att, const ton_api::validatorSession_message_voteFor& act, const ValidatorSessionRoundState* round) { - if (state->vote_for_inited_) { - VLOG(VALIDATOR_SESSION_WARNING) << "[validator session][node " << desc.get_source_id(src_idx) << "][" << act - << "]: invalid message: duplicate VOTEFOR"; - return state; - } - if (src_idx != desc.get_vote_for_author(att)) { - VLOG(VALIDATOR_SESSION_WARNING) << "[validator session][node " << desc.get_source_id(src_idx) << "][" << act - << "]: invalid message: bad VOTEFOR author"; - return state; - } - if (round->get_first_attempt(src_idx) == 0 && desc.opts().max_round_attempts > 0) { - VLOG(VALIDATOR_SESSION_WARNING) << "[validator session][node " << desc.get_source_id(src_idx) << "][" << act - << "]: invalid message: too early for VOTEFOR"; - return state; - } - if (round->get_first_attempt(src_idx) + desc.opts().max_round_attempts > att && desc.opts().max_round_attempts == 0) { - VLOG(VALIDATOR_SESSION_WARNING) << "[validator session][node " << desc.get_source_id(src_idx) << "][" << act - << "]: invalid message: too early for VOTEFOR"; - return state; - } - - auto x = round->get_block(act.candidate_); - - if (!x) { - VLOG(VALIDATOR_SESSION_WARNING) << "[validator session][node " << desc.get_source_id(src_idx) << "][" << act - << "]: invalid message: VOTEFOR for not approved block"; - return state; - } - if (!x->check_block_is_approved(desc)) { - VLOG(VALIDATOR_SESSION_WARNING) << "[validator session][node " << desc.get_source_id(src_idx) << "][" << act - << "]: invalid message: VOTEFOR for not approved block"; - return state; - } - - return ValidatorSessionRoundAttemptState::create(desc, state->seqno_, state->votes_, state->precommitted_, - x->get_block(), true); -} - -const ValidatorSessionRoundAttemptState* ValidatorSessionRoundAttemptState::action( - ValidatorSessionDescription& desc, const ValidatorSessionRoundAttemptState* state, td::uint32 src_idx, - td::uint32 att, const ton_api::validatorSession_message_vote& act, const ValidatorSessionRoundState* round) { - bool made; - return make_one(desc, state, src_idx, att, round, &act, made); -} - -const ValidatorSessionRoundAttemptState* ValidatorSessionRoundAttemptState::action( - ValidatorSessionDescription& desc, const ValidatorSessionRoundAttemptState* state, td::uint32 src_idx, - td::uint32 att, const ton_api::validatorSession_message_precommit& act, const ValidatorSessionRoundState* round) { - bool made; - return make_one(desc, state, src_idx, att, round, &act, made); -} - -const ValidatorSessionRoundAttemptState* ValidatorSessionRoundAttemptState::action( - ValidatorSessionDescription& desc, const ValidatorSessionRoundAttemptState* state, td::uint32 src_idx, - td::uint32 att, const ton_api::validatorSession_message_empty& act, const ValidatorSessionRoundState* round) { - bool made; - return make_one(desc, state, src_idx, att, round, &act, made); -} - -const ValidatorSessionRoundAttemptState* ValidatorSessionRoundAttemptState::try_vote( - ValidatorSessionDescription& desc, const ValidatorSessionRoundAttemptState* state, td::uint32 src_idx, - td::uint32 att, const ValidatorSessionRoundState* round, const ton_api::validatorSession_round_Message* act, - bool& made) { - made = false; - if (state->check_vote_received_from(src_idx)) { - return state; - } - auto found = false; - auto block = round->choose_block_to_vote(desc, src_idx, att, state->vote_for_, state->vote_for_inited_, found); - if (!found) { - return state; - } - auto block_id = SentBlock::get_block_id(block); - made = true; - - if (act) { - if (act->get_id() != ton_api::validatorSession_message_vote::ID) { - VLOG(VALIDATOR_SESSION_WARNING) << "[validator session][node " << desc.get_source_id(src_idx) << "][" << act - << "]: expected VOTE(" << block_id << ")"; - } else { - auto x = static_cast(act); - if (x->candidate_ != block_id) { - VLOG(VALIDATOR_SESSION_WARNING) << "[validator session][node " << desc.get_source_id(src_idx) << "][" << act - << "]: expected VOTE(" << block_id << ")"; - } - } - } else { - VLOG(VALIDATOR_SESSION_WARNING) << "[validator session][node " << desc.get_source_id(src_idx) - << "]: making implicit VOTE(" << block_id << ")"; - } - - auto candidate = get_candidate(state->votes_, block_id); - if (!candidate) { - candidate = SessionVoteCandidate::create(desc, block); - } - candidate = SessionVoteCandidate::push(desc, candidate, src_idx); - auto v = VoteVector::push(desc, state->votes_, candidate); - return ValidatorSessionRoundAttemptState::create(desc, state->seqno_, std::move(v), state->precommitted_, - state->vote_for_, state->vote_for_inited_); -} - -const ValidatorSessionRoundAttemptState* ValidatorSessionRoundAttemptState::try_precommit( - ValidatorSessionDescription& desc, const ValidatorSessionRoundAttemptState* state, td::uint32 src_idx, - td::uint32 att, const ValidatorSessionRoundState* round, const ton_api::validatorSession_round_Message* act, - bool& made) { - made = false; - if (state->check_precommit_received_from(src_idx)) { - return state; - } - bool found; - auto block = state->get_voted_block(desc, found); - if (!found) { - return state; - } - made = true; - auto block_id = SentBlock::get_block_id(block); - - if (act) { - if (act->get_id() != ton_api::validatorSession_message_precommit::ID) { - VLOG(VALIDATOR_SESSION_WARNING) << "[validator session][node " << desc.get_source_id(src_idx) << "][" << act - << "]: expected PRECOMMIT(" << block_id << ")"; - } else { - auto x = static_cast(act); - if (x->candidate_ != block_id) { - VLOG(VALIDATOR_SESSION_WARNING) << "[validator session][node " << desc.get_source_id(src_idx) << "][" << act - << "]: expected PRECOMMIT(" << block_id << ")"; - } - } - } else { - VLOG(VALIDATOR_SESSION_WARNING) << "[validator session][node " << desc.get_source_id(src_idx) - << "]: making implicit PRECOMMIT(" << block_id << ")"; - } - - auto v = CntVector::change(desc, state->precommitted_, src_idx, true); - return ValidatorSessionRoundAttemptState::create(desc, state->seqno_, state->votes_, std::move(v), state->vote_for_, - state->vote_for_inited_); -} - -const ValidatorSessionRoundAttemptState* ValidatorSessionRoundAttemptState::make_one( - ValidatorSessionDescription& desc, const ValidatorSessionRoundAttemptState* state, td::uint32 src_idx, - td::uint32 att, const ValidatorSessionRoundState* round, const ton_api::validatorSession_round_Message* act, - bool& made) { - made = false; - state = try_vote(desc, state, src_idx, att, round, act, made); - if (made) { - return state; - } - state = try_precommit(desc, state, src_idx, att, round, act, made); - if (made) { - return state; - } - if (act && act->get_id() != ton_api::validatorSession_message_empty::ID) { - VLOG(VALIDATOR_SESSION_WARNING) << "[validator session][node " << desc.get_source_id(src_idx) << "][" << act - << "]: invalid message: expected EMPTY"; - } - return state; -} - -const ValidatorSessionRoundAttemptState* ValidatorSessionRoundAttemptState::action( - ValidatorSessionDescription& desc, const ValidatorSessionRoundAttemptState* state, td::uint32 src_idx, - td::uint32 att, const ton_api::validatorSession_round_Message* act, const ValidatorSessionRoundState* round) { - ton_api::downcast_call(*const_cast(act), - [&](auto& obj) { state = action(desc, state, src_idx, att, obj, round); }); - return state; -} - -bool ValidatorSessionRoundAttemptState::check_vote_received_from(td::uint32 src_idx) const { - if (!votes_) { - return false; - } - auto size = votes_->size(); - auto v = votes_->data(); - for (td::uint32 i = 0; i < size; i++) { - if (v[i]->check_block_is_voted_by(src_idx)) { - return true; - } - } - return false; -} - -bool ValidatorSessionRoundAttemptState::check_precommit_received_from(td::uint32 src_idx) const { - return precommitted_->at(src_idx); -} - -const SentBlock* ValidatorSessionRoundAttemptState::get_voted_block(ValidatorSessionDescription& desc, bool& f) const { - f = false; - if (!votes_) { - return nullptr; - } - - auto size = votes_->size(); - auto v = votes_->data(); - for (td::uint32 i = 0; i < size; i++) { - if (v[i]->check_block_is_voted(desc)) { - f = true; - return v[i]->get_block(); - } - } - return nullptr; -} - -bool ValidatorSessionRoundAttemptState::check_attempt_is_precommitted(ValidatorSessionDescription& desc) const { - ValidatorWeight weight = 0; - for (td::uint32 i = 0; i < desc.get_total_nodes(); i++) { - if (precommitted_->at(i)) { - weight += desc.get_node_weight(i); - if (weight >= desc.get_cutoff_weight()) { - return true; - } - } - } - return false; -} - -tl_object_ptr ValidatorSessionRoundAttemptState::create_action( - ValidatorSessionDescription& desc, const ValidatorSessionRoundState* round, td::uint32 src_idx, - td::uint32 att) const { - if (!check_vote_received_from(src_idx)) { - auto found = false; - auto B = round->choose_block_to_vote(desc, src_idx, att, vote_for_, vote_for_inited_, found); - if (found) { - auto block_id = SentBlock::get_block_id(B); - return create_tl_object(round->get_seqno(), seqno_, block_id); - } - } - if (!check_precommit_received_from(src_idx)) { - bool f = false; - auto B = get_voted_block(desc, f); - - if (f) { - auto block_id = SentBlock::get_block_id(B); - - return create_tl_object(round->get_seqno(), seqno_, block_id); - } - } - - return create_tl_object(round->get_seqno(), seqno_); -} - -void ValidatorSessionRoundAttemptState::dump(ValidatorSessionDescription& desc, td::StringBuilder& sb) const { - sb << "attempt=" << seqno_ << "\n"; - sb << ">>>>\n"; - - if (vote_for_inited_) { - sb << "vote_for=" << (vote_for_ ? vote_for_->get_src_idx() : std::numeric_limits::max()) << "\n"; - } else { - sb << "vote_for=NONE\n"; - } - - if (votes_) { - auto s = votes_->size(); - sb << "votes: "; - std::vector R; - R.resize(desc.get_total_nodes(), -1); - for (td::uint32 i = 0; i < s; i++) { - const auto e = votes_->at(i); - const auto& x = e->get_voters_list(); - for (td::uint32 j = 0; j < desc.get_total_nodes(); j++) { - if (x->at(j)) { - R[j] = e->get_src_idx(); - } - } - } - for (td::uint32 i = 0; i < desc.get_total_nodes(); i++) { - sb << R[i] << " "; - } - sb << "\n"; - } else { - sb << "votes: EMPTY\n"; - } - - sb << "precommits: "; - for (td::uint32 i = 0; i < desc.get_total_nodes(); i++) { - const auto e = precommitted_->at(i); - if (e) { - sb << "+ "; - } else { - sb << "- "; - } - } - sb << "\n"; - sb << "<<<<\n"; -} - // // // ROUND STATE @@ -821,8 +367,6 @@ const ValidatorSessionRoundState* ValidatorSessionRoundState::merge(ValidatorSes return SessionBlockCandidateSignature::merge(desc, l, r); }); - //auto sent_vec = SentBlockVector::merge(desc, left->sent_blocks_, right->sent_blocks_); - auto sent = ApproveVector::merge(desc, left->sent_blocks_, right->sent_blocks_, [&](const SessionBlockCandidate* l, const SessionBlockCandidate* r) { return SessionBlockCandidate::merge(desc, l, r); @@ -985,9 +529,6 @@ const ValidatorSessionRoundState* ValidatorSessionRoundState::forward_action_to_ attempt = ValidatorSessionRoundAttemptState::create(desc, att); } - bool had_voted_block; - attempt->get_voted_block(desc, had_voted_block); - ton_api::downcast_call(*const_cast(act), [&](auto& obj) { attempt = ValidatorSessionRoundAttemptState::action(desc, attempt, src_idx, att, obj, state); }); @@ -1302,6 +843,8 @@ std::vector ValidatorSessionRoundState::choose_blocks_to_appro CHECK(prio >= 0); td::uint32 blk_src_idx = B->get_src_idx(); if (was_source.count(blk_src_idx) > 0) { + // Any honest validator submits at most one block in a round + // Therefore, we can ignore all blocks from a node if it submits more than one x[prio] = nullptr; } else { was_source.insert(blk_src_idx); diff --git a/validator-session/validator-session-state.h b/validator-session/validator-session-state.h index 37708cba9..f827b3fcb 100644 --- a/validator-session/validator-session-state.h +++ b/validator-session/validator-session-state.h @@ -24,15 +24,15 @@ #include "common/io.hpp" #include "persistent-vector.h" - #include "validator-session-description.h" - #include "validator-session-types.h" +#include "validator-session-round-attempt-state.h" #include namespace td { +td::StringBuilder& operator<<(td::StringBuilder& sb, const ton::ton_api::validatorSession_round_Message& message); td::StringBuilder& operator<<(td::StringBuilder& sb, const ton::ton_api::validatorSession_round_Message* message); } @@ -41,410 +41,6 @@ namespace ton { namespace validatorsession { -using HashType = ValidatorSessionDescription::HashType; - -constexpr int VERBOSITY_NAME(VALIDATOR_SESSION_WARNING) = verbosity_WARNING; -constexpr int VERBOSITY_NAME(VALIDATOR_SESSION_NOTICE) = verbosity_DEBUG; -constexpr int VERBOSITY_NAME(VALIDATOR_SESSION_INFO) = verbosity_DEBUG; -constexpr int VERBOSITY_NAME(VALIDATOR_SESSION_DEBUG) = verbosity_DEBUG; -constexpr int VERBOSITY_NAME(VALIDATOR_SESSION_EXTRA_DEBUG) = verbosity_DEBUG + 1; - -struct SessionBlockCandidateSignature : public ValidatorSessionDescription::RootObject { - public: - static auto create_hash(ValidatorSessionDescription& desc, td::Slice data) { - auto obj = create_tl_object(desc.compute_hash(data)); - return desc.compute_hash(serialize_tl_object(obj, true).as_slice()); - } - - static bool compare(const RootObject* r, td::Slice data, HashType hash) { - if (!r || r->get_size() < sizeof(SessionBlockCandidateSignature)) { - return false; - } - auto R = static_cast(r); - return R->hash_ == hash && R->data_.ubegin() == data.ubegin() && R->data_.size() == data.size(); - } - - static auto lookup(ValidatorSessionDescription& desc, td::Slice data, HashType hash, bool temp) { - auto r = desc.get_by_hash(hash, temp); - if (compare(r, data, hash)) { - desc.on_reuse(); - return static_cast(r); - } - return static_cast(nullptr); - } - static SessionBlockCandidateSignature* create(ValidatorSessionDescription& desc, td::BufferSlice value) { - auto hash = create_hash(desc, value.as_slice()); - auto d = static_cast(desc.alloc(value.size(), 8, false)); - td::MutableSlice s{d, value.size()}; - s.copy_from(value.as_slice()); - return new (desc, true) SessionBlockCandidateSignature{desc, s, hash}; - } - static const SessionBlockCandidateSignature* move_to_persistent(ValidatorSessionDescription& desc, - const SessionBlockCandidateSignature* b) { - if (desc.is_persistent(b)) { - return b; - } - CHECK(desc.is_persistent(b->data_.ubegin())); - auto r = lookup(desc, b->data_, b->hash_, false); - if (r) { - return r; - } - return new (desc, false) SessionBlockCandidateSignature{desc, b->data_, b->hash_}; - } - static const SessionBlockCandidateSignature* merge(ValidatorSessionDescription& desc, - const SessionBlockCandidateSignature* l, - const SessionBlockCandidateSignature* r); - SessionBlockCandidateSignature(ValidatorSessionDescription& desc, td::Slice data, HashType hash) - : RootObject(sizeof(SessionBlockCandidateSignature)), data_{data}, hash_(std::move(hash)) { - desc.update_hash(this, hash_); - } - td::BufferSlice value() const { - return td::BufferSlice{data_}; - } - td::Slice as_slice() const { - return data_; - } - auto get_hash(ValidatorSessionDescription& desc) const { - return hash_; - } - - private: - const td::Slice data_; - const HashType hash_; -}; - -using SessionBlockCandidateSignatureVector = CntVector; - -class SentBlock : public ValidatorSessionDescription::RootObject { - public: - static HashType create_hash(ValidatorSessionDescription& desc, td::uint32 src_idx, ValidatorSessionRootHash root_hash, - ValidatorSessionFileHash file_hash, - ValidatorSessionCollatedDataFileHash collated_data_file_hash) { - auto obj = create_tl_object(src_idx, get_vs_hash(desc, root_hash), - get_vs_hash(desc, file_hash), - get_vs_hash(desc, collated_data_file_hash)); - return desc.compute_hash(serialize_tl_object(obj, true).as_slice()); - } - static bool compare(const RootObject* root_object, td::uint32 src_idx, const ValidatorSessionRootHash& root_hash, - const ValidatorSessionFileHash& file_hash, - const ValidatorSessionCollatedDataFileHash& collated_data_file_hash, HashType hash) { - if (!root_object || root_object->get_size() < sizeof(SentBlock)) { - return false; - } - auto obj = static_cast(root_object); - return obj->src_idx_ == src_idx && obj->root_hash_ == root_hash && obj->file_hash_ == file_hash && - obj->collated_data_file_hash_ == collated_data_file_hash && obj->hash_ == hash; - } - static auto lookup(ValidatorSessionDescription& desc, td::uint32 src_idx, const ValidatorSessionRootHash& root_hash, - const ValidatorSessionFileHash& file_hash, - const ValidatorSessionCollatedDataFileHash& collated_data_file_hash, HashType hash, bool temp) { - auto r = desc.get_by_hash(hash, temp); - if (compare(r, src_idx, root_hash, file_hash, collated_data_file_hash, hash)) { - desc.on_reuse(); - return static_cast(r); - } - return static_cast(nullptr); - } - static const SentBlock* create(ValidatorSessionDescription& desc, td::uint32 src_idx, - const ValidatorSessionRootHash& root_hash, const ValidatorSessionFileHash& file_hash, - const ValidatorSessionCollatedDataFileHash& collated_data_file_hash) { - auto hash = create_hash(desc, src_idx, root_hash, file_hash, collated_data_file_hash); - auto r = lookup(desc, src_idx, root_hash, file_hash, collated_data_file_hash, hash, true); - if (r) { - return r; - } - - auto candidate_id = desc.candidate_id(src_idx, root_hash, file_hash, collated_data_file_hash); - - return new (desc, true) SentBlock{desc, src_idx, root_hash, file_hash, collated_data_file_hash, candidate_id, hash}; - } - static const SentBlock* create(ValidatorSessionDescription& desc, const ValidatorSessionCandidateId& zero) { - CHECK(zero.is_zero()); - auto hash = create_hash(desc, 0, ValidatorSessionRootHash::zero(), ValidatorSessionFileHash::zero(), - ValidatorSessionCollatedDataFileHash::zero()); - - return new (desc, true) SentBlock{desc, - 0, - ValidatorSessionRootHash::zero(), - ValidatorSessionFileHash::zero(), - ValidatorSessionCollatedDataFileHash::zero(), - zero, - hash}; - } - static const SentBlock* move_to_persistent(ValidatorSessionDescription& desc, const SentBlock* b) { - if (desc.is_persistent(b)) { - return b; - } - auto r = lookup(desc, b->src_idx_, b->root_hash_, b->file_hash_, b->collated_data_file_hash_, b->hash_, false); - if (r) { - return r; - } - - return new (desc, false) SentBlock{ - desc, b->src_idx_, b->root_hash_, b->file_hash_, b->collated_data_file_hash_, b->candidate_id_, b->hash_}; - } - SentBlock(ValidatorSessionDescription& desc, td::uint32 src_idx, ValidatorSessionRootHash root_hash, - ValidatorSessionFileHash file_hash, ValidatorSessionCollatedDataFileHash collated_data_file_hash, - ValidatorSessionCandidateId candidate_id, HashType hash) - : RootObject(sizeof(SentBlock)) - , src_idx_(src_idx) - , root_hash_(std::move(root_hash)) - , file_hash_(std::move(file_hash)) - , collated_data_file_hash_(std::move(collated_data_file_hash)) - , candidate_id_(candidate_id) - , hash_(std::move(hash)) { - desc.update_hash(this, hash_); - } - auto get_src_idx() const { - return src_idx_; - } - auto get_root_hash() const { - return root_hash_; - } - auto get_file_hash() const { - return file_hash_; - } - auto get_collated_data_file_hash() const { - return collated_data_file_hash_; - } - static ValidatorSessionCandidateId get_block_id(const SentBlock* block) { - return block ? block->candidate_id_ : skip_round_candidate_id(); - } - HashType get_hash(ValidatorSessionDescription& desc) const { - return hash_; - } - bool operator<(const SentBlock& block) const { - if (src_idx_ < block.src_idx_) { - return true; - } - if (src_idx_ > block.src_idx_) { - return false; - } - if (candidate_id_ < block.candidate_id_) { - return true; - } - return false; - } - struct Compare { - bool operator()(const SentBlock* a, const SentBlock* b) const { - return *a < *b; - } - }; - - private: - const td::uint32 src_idx_; - const ValidatorSessionRootHash root_hash_; - const ValidatorSessionFileHash file_hash_; - const ValidatorSessionCollatedDataFileHash collated_data_file_hash_; - const ValidatorSessionCandidateId candidate_id_; - const HashType hash_; -}; - -class SessionBlockCandidate : public ValidatorSessionDescription::RootObject { - public: - static HashType create_hash(ValidatorSessionDescription& desc, HashType block, HashType approved) { - auto obj = create_tl_object(block, approved); - return desc.compute_hash(serialize_tl_object(obj, true).as_slice()); - } - static bool compare(const RootObject* r, const SentBlock* block, const SessionBlockCandidateSignatureVector* approved, - HashType hash) { - if (!r || r->get_size() < sizeof(SessionBlockCandidate)) { - return false; - } - auto R = static_cast(r); - return R->block_ == block && R->approved_by_ == approved && R->hash_ == hash; - } - static auto lookup(ValidatorSessionDescription& desc, const SentBlock* block, - const SessionBlockCandidateSignatureVector* approved, HashType hash, bool temp) { - auto r = desc.get_by_hash(hash, temp); - if (compare(r, block, approved, hash)) { - desc.on_reuse(); - return static_cast(r); - } - return static_cast(nullptr); - } - static const SessionBlockCandidate* create(ValidatorSessionDescription& desc, const SentBlock* block, - const SessionBlockCandidateSignatureVector* approved) { - auto hash = create_hash(desc, get_vs_hash(desc, block), get_vs_hash(desc, approved)); - - auto r = lookup(desc, block, approved, hash, true); - if (r) { - return r; - } - - return new (desc, true) SessionBlockCandidate(desc, block, approved, hash); - } - static const SessionBlockCandidate* create(ValidatorSessionDescription& desc, const SentBlock* block) { - std::vector v; - v.resize(desc.get_total_nodes(), nullptr); - auto vec = SessionBlockCandidateSignatureVector::create(desc, std::move(v)); - return create(desc, block, vec); - } - static const SessionBlockCandidate* move_to_persistent(ValidatorSessionDescription& desc, - const SessionBlockCandidate* b) { - if (desc.is_persistent(b)) { - return b; - } - auto block = SentBlock::move_to_persistent(desc, b->block_); - auto approved = SessionBlockCandidateSignatureVector::move_to_persistent(desc, b->approved_by_); - auto r = lookup(desc, block, approved, b->hash_, false); - if (r) { - return r; - } - - return new (desc, false) SessionBlockCandidate{desc, block, approved, b->hash_}; - } - SessionBlockCandidate(ValidatorSessionDescription& desc, const SentBlock* block, - const SessionBlockCandidateSignatureVector* approved, HashType hash) - : RootObject{sizeof(SessionBlockCandidate)}, block_(block), approved_by_(approved), hash_(hash) { - desc.update_hash(this, hash_); - } - static const SessionBlockCandidate* merge(ValidatorSessionDescription& desc, const SessionBlockCandidate* l, - const SessionBlockCandidate* r); - auto get_block() const { - return block_; - } - auto get_id() const { - return SentBlock::get_block_id(block_); - } - auto get_src_idx() const { - return block_ ? block_->get_src_idx() : std::numeric_limits::max(); - } - bool check_block_is_approved_by(td::uint32 src_idx) const { - return approved_by_->at(src_idx); - } - bool check_block_is_approved(ValidatorSessionDescription& desc) const; - auto get_hash(ValidatorSessionDescription& desc) const { - return hash_; - } - auto get_approvers_list() const { - return approved_by_; - } - static const SessionBlockCandidate* push(ValidatorSessionDescription& desc, const SessionBlockCandidate* state, - td::uint32 src_idx, const SessionBlockCandidateSignature* sig) { - CHECK(state); - if (state->approved_by_->at(src_idx)) { - return state; - } - return create(desc, state->block_, - SessionBlockCandidateSignatureVector::change(desc, state->approved_by_, src_idx, sig)); - } - class Compare { - public: - bool operator()(const SessionBlockCandidate* l, const SessionBlockCandidate* r) { - return l->get_id() < r->get_id(); - } - }; - - private: - const SentBlock* block_; - const SessionBlockCandidateSignatureVector* approved_by_; - const HashType hash_; -}; - -class SessionVoteCandidate : public ValidatorSessionDescription::RootObject { - public: - static HashType create_hash(ValidatorSessionDescription& desc, HashType block, HashType voted) { - auto obj = create_tl_object(block, voted); - return desc.compute_hash(serialize_tl_object(obj, true).as_slice()); - } - static bool compare(const RootObject* r, const SentBlock* block, const CntVector* voted, HashType hash) { - if (!r || r->get_size() < sizeof(SessionVoteCandidate)) { - return false; - } - auto R = static_cast(r); - return R->block_ == block && R->voted_by_ == voted && R->hash_ == hash; - } - static auto lookup(ValidatorSessionDescription& desc, const SentBlock* block, const CntVector* voted, - HashType hash, bool temp) { - auto r = desc.get_by_hash(hash, temp); - if (compare(r, block, voted, hash)) { - desc.on_reuse(); - return static_cast(r); - } - return static_cast(nullptr); - } - static const SessionVoteCandidate* create(ValidatorSessionDescription& desc, const SentBlock* block, - const CntVector* voted) { - auto hash = create_hash(desc, get_vs_hash(desc, block), get_vs_hash(desc, voted)); - - auto r = lookup(desc, block, voted, hash, true); - if (r) { - return r; - } - - return new (desc, true) SessionVoteCandidate(desc, block, voted, hash); - } - static const SessionVoteCandidate* create(ValidatorSessionDescription& desc, const SentBlock* block) { - std::vector v; - v.resize(desc.get_total_nodes(), false); - auto vec = CntVector::create(desc, std::move(v)); - return create(desc, block, vec); - } - static const SessionVoteCandidate* move_to_persistent(ValidatorSessionDescription& desc, - const SessionVoteCandidate* b) { - if (desc.is_persistent(b)) { - return b; - } - auto block = SentBlock::move_to_persistent(desc, b->block_); - auto voted = CntVector::move_to_persistent(desc, b->voted_by_); - auto r = lookup(desc, block, voted, b->hash_, false); - if (r) { - return r; - } - - return new (desc, false) SessionVoteCandidate{desc, block, voted, b->hash_}; - } - SessionVoteCandidate(ValidatorSessionDescription& desc, const SentBlock* block, const CntVector* voted, - HashType hash) - : RootObject{sizeof(SessionVoteCandidate)}, block_(block), voted_by_(voted), hash_(hash) { - desc.update_hash(this, hash_); - } - static const SessionVoteCandidate* merge(ValidatorSessionDescription& desc, const SessionVoteCandidate* l, - const SessionVoteCandidate* r); - auto get_block() const { - return block_; - } - auto get_id() const { - return SentBlock::get_block_id(block_); - } - auto get_src_idx() const { - return block_ ? block_->get_src_idx() : std::numeric_limits::max(); - } - bool check_block_is_voted_by(td::uint32 src_idx) const { - return voted_by_->at(src_idx); - } - bool check_block_is_voted(ValidatorSessionDescription& desc) const; - auto get_hash(ValidatorSessionDescription& desc) const { - return hash_; - } - auto get_voters_list() const { - return voted_by_; - } - static const SessionVoteCandidate* push(ValidatorSessionDescription& desc, const SessionVoteCandidate* state, - td::uint32 src_idx) { - CHECK(state); - if (state->voted_by_->at(src_idx)) { - return state; - } - return create(desc, state->block_, CntVector::change(desc, state->voted_by_, src_idx, true)); - } - class Compare { - public: - bool operator()(const SessionVoteCandidate* l, const SessionVoteCandidate* r) { - return l->get_id() < r->get_id(); - } - }; - - private: - const SentBlock* block_; - const CntVector* voted_by_; - const HashType hash_; -}; - -//using SentBlockVector = CntSortedVector; - -class ValidatorSessionRoundState; class ValidatorSessionOldRoundState : public ValidatorSessionDescription::RootObject { public: static HashType create_hash(ValidatorSessionDescription& desc, td::uint32 seqno, HashType block, HashType signatures, @@ -582,188 +178,6 @@ class ValidatorSessionOldRoundState : public ValidatorSessionDescription::RootOb const HashType hash_; }; -using VoteVector = CntSortedVector; -using ApproveVector = CntSortedVector; - -class ValidatorSessionRoundAttemptState : public ValidatorSessionDescription::RootObject { - public: - static HashType create_hash(ValidatorSessionDescription& desc, td::uint32 seqno, HashType votes, - HashType precommitted, bool vote_for_inited, HashType vote_for) { - auto obj = create_tl_object(seqno, votes, precommitted, - vote_for_inited, vote_for); - return desc.compute_hash(serialize_tl_object(obj, true).as_slice()); - } - static bool compare(const RootObject* r, td::uint32 seqno, const VoteVector* votes, - const CntVector* precommitted, const SentBlock* vote_for, bool vote_for_inited, - HashType hash) { - if (!r || r->get_size() < sizeof(ValidatorSessionRoundAttemptState)) { - return false; - } - auto R = static_cast(r); - return R->seqno_ == seqno && R->votes_ == votes && R->precommitted_ == precommitted && R->vote_for_ == vote_for && - R->vote_for_inited_ == vote_for_inited && R->hash_ == hash; - } - static auto lookup(ValidatorSessionDescription& desc, td::uint32 seqno, const VoteVector* votes, - const CntVector* precommitted, const SentBlock* vote_for, bool vote_for_inited, - HashType hash, bool temp) { - auto r = desc.get_by_hash(hash, temp); - if (compare(r, seqno, votes, precommitted, vote_for, vote_for_inited, hash)) { - desc.on_reuse(); - return static_cast(r); - } - return static_cast(nullptr); - } - static const ValidatorSessionRoundAttemptState* move_to_persistent(ValidatorSessionDescription& desc, - const ValidatorSessionRoundAttemptState* b) { - if (desc.is_persistent(b)) { - return b; - } - auto votes = VoteVector::move_to_persistent(desc, b->votes_); - auto precommitted = CntVector::move_to_persistent(desc, b->precommitted_); - auto vote_for = SentBlock::move_to_persistent(desc, b->vote_for_); - - auto r = lookup(desc, b->seqno_, votes, precommitted, vote_for, b->vote_for_inited_, b->hash_, false); - if (r) { - return r; - } - - return new (desc, false) ValidatorSessionRoundAttemptState{desc, b->seqno_, votes, precommitted, - vote_for, b->vote_for_inited_, b->hash_}; - } - - static const ValidatorSessionRoundAttemptState* create(ValidatorSessionDescription& desc, td::uint32 seqno, - const VoteVector* votes, const CntVector* precommitted, - const SentBlock* vote_for, bool vote_for_inited) { - auto hash = create_hash(desc, seqno, get_vs_hash(desc, votes), get_vs_hash(desc, precommitted), - get_vs_hash(desc, vote_for), vote_for_inited); - - auto r = lookup(desc, seqno, votes, precommitted, vote_for, vote_for_inited, hash, true); - if (r) { - return r; - } - - return new (desc, true) - ValidatorSessionRoundAttemptState(desc, seqno, votes, precommitted, vote_for, vote_for_inited, hash); - } - static const ValidatorSessionRoundAttemptState* create(ValidatorSessionDescription& desc, td::uint32 seqno) { - std::vector x; - x.resize(desc.get_total_nodes(), false); - auto p = CntVector::create(desc, std::move(x)); - - return create(desc, seqno, nullptr, p, nullptr, false); - } - - ValidatorSessionRoundAttemptState(ValidatorSessionDescription& desc, td::uint32 seqno, const VoteVector* votes, - const CntVector* precommitted, const SentBlock* vote_for, - bool vote_for_inited, HashType hash) - : RootObject{sizeof(ValidatorSessionRoundAttemptState)} - , seqno_(seqno) - , votes_(votes) - , precommitted_(precommitted) - , vote_for_(vote_for) - , vote_for_inited_(vote_for_inited) - , hash_(std::move(hash)) { - desc.update_hash(this, hash_); - } - auto get_hash(ValidatorSessionDescription& desc) const { - return hash_; - } - auto get_seqno() const { - return seqno_; - } - auto get_votes() const { - return votes_; - } - auto get_precommits() const { - return precommitted_; - } - const SentBlock* get_voted_block(ValidatorSessionDescription& desc, bool& f) const; - const SentBlock* get_vote_for_block(ValidatorSessionDescription& desc, bool& f) const { - f = vote_for_inited_; - return vote_for_; - } - bool check_attempt_is_precommitted(ValidatorSessionDescription& desc) const; - bool check_vote_received_from(td::uint32 src_idx) const; - bool check_precommit_received_from(td::uint32 src_idx) const; - - bool operator<(const ValidatorSessionRoundAttemptState& right) const { - return seqno_ < right.seqno_; - } - struct Compare { - bool operator()(const ValidatorSessionRoundAttemptState* a, const ValidatorSessionRoundAttemptState* b) const { - return *a < *b; - } - }; - - static const ValidatorSessionRoundAttemptState* merge(ValidatorSessionDescription& desc, - const ValidatorSessionRoundAttemptState* left, - const ValidatorSessionRoundAttemptState* right); - static const ValidatorSessionRoundAttemptState* action(ValidatorSessionDescription& desc, - const ValidatorSessionRoundAttemptState* state, - td::uint32 src_idx, td::uint32 att, - const ton_api::validatorSession_message_voteFor& act, - const ValidatorSessionRoundState* round); - static const ValidatorSessionRoundAttemptState* action(ValidatorSessionDescription& desc, - const ValidatorSessionRoundAttemptState* state, - td::uint32 src_idx, td::uint32 att, - const ton_api::validatorSession_message_vote& act, - const ValidatorSessionRoundState* round); - static const ValidatorSessionRoundAttemptState* action(ValidatorSessionDescription& desc, - const ValidatorSessionRoundAttemptState* state, - td::uint32 src_idx, td::uint32 att, - const ton_api::validatorSession_message_precommit& act, - const ValidatorSessionRoundState* round); - static const ValidatorSessionRoundAttemptState* action(ValidatorSessionDescription& desc, - const ValidatorSessionRoundAttemptState* state, - td::uint32 src_idx, td::uint32 att, - const ton_api::validatorSession_message_empty& act, - const ValidatorSessionRoundState* round); - static const ValidatorSessionRoundAttemptState* action(ValidatorSessionDescription& desc, - const ValidatorSessionRoundAttemptState* state, - td::uint32 src_idx, td::uint32 att, - const ton_api::validatorSession_round_Message* action, - const ValidatorSessionRoundState* round); - template - static const ValidatorSessionRoundAttemptState* action(ValidatorSessionDescription& desc, - const ValidatorSessionRoundAttemptState* state, - td::uint32 src_idx, td::uint32 att, const T& action, - const ValidatorSessionRoundState* round) { - UNREACHABLE(); - } - static const ValidatorSessionRoundAttemptState* try_vote(ValidatorSessionDescription& desc, - const ValidatorSessionRoundAttemptState* state, - td::uint32 src_idx, td::uint32 att, - const ValidatorSessionRoundState* round, - const ton_api::validatorSession_round_Message* cmp, - bool& made); - static const ValidatorSessionRoundAttemptState* try_precommit(ValidatorSessionDescription& desc, - const ValidatorSessionRoundAttemptState* state, - td::uint32 src_idx, td::uint32 att, - const ValidatorSessionRoundState* round, - const ton_api::validatorSession_round_Message* cmp, - bool& made); - static const ValidatorSessionRoundAttemptState* make_one(ValidatorSessionDescription& desc, - const ValidatorSessionRoundAttemptState* state, - td::uint32 src_idx, td::uint32 att, - const ValidatorSessionRoundState* round, - const ton_api::validatorSession_round_Message* cmp, - bool& made); - tl_object_ptr create_action(ValidatorSessionDescription& desc, - const ValidatorSessionRoundState* round, - td::uint32 src_idx, td::uint32 att) const; - void dump(ValidatorSessionDescription& desc, td::StringBuilder& sb) const; - - private: - const td::uint32 seqno_; - const VoteVector* votes_; - const CntVector* precommitted_; - const SentBlock* vote_for_; - const bool vote_for_inited_; - const HashType hash_; -}; - -using AttemptVector = - CntSortedVector; class ValidatorSessionRoundState : public ValidatorSessionDescription::RootObject { public: diff --git a/validator-session/validator-session-types.h b/validator-session/validator-session-types.h index 357f6217b..579574786 100644 --- a/validator-session/validator-session-types.h +++ b/validator-session/validator-session-types.h @@ -27,6 +27,12 @@ namespace ton { namespace validatorsession { +constexpr int VERBOSITY_NAME(VALIDATOR_SESSION_WARNING) = verbosity_WARNING; +constexpr int VERBOSITY_NAME(VALIDATOR_SESSION_NOTICE) = verbosity_DEBUG; +constexpr int VERBOSITY_NAME(VALIDATOR_SESSION_INFO) = verbosity_DEBUG; +constexpr int VERBOSITY_NAME(VALIDATOR_SESSION_DEBUG) = verbosity_DEBUG; +constexpr int VERBOSITY_NAME(VALIDATOR_SESSION_EXTRA_DEBUG) = verbosity_DEBUG + 1; + using ValidatorSessionRootHash = td::Bits256; using ValidatorSessionFileHash = td::Bits256; using ValidatorSessionCollatedDataFileHash = td::Bits256; diff --git a/validator-session/validator-session.cpp b/validator-session/validator-session.cpp index 2902b082c..e08d8a7e0 100644 --- a/validator-session/validator-session.cpp +++ b/validator-session/validator-session.cpp @@ -203,7 +203,23 @@ void ValidatorSessionImpl::preprocess_block(catchain::CatChainBlock *block) { << "ms: state=" << state->get_hash(description()); } -void ValidatorSessionImpl::process_broadcast(PublicKeyHash src, td::BufferSlice data) { +bool ValidatorSessionImpl::ensure_candidate_unique(td::uint32 src_idx, td::uint32 round, + ValidatorSessionCandidateId block_id) { + auto it = src_round_candidate_[src_idx].find(round); + if (it != src_round_candidate_[src_idx].end() && it->second != block_id) { + VLOG(VALIDATOR_SESSION_WARNING) << this << "[node " << description_->get_source_adnl_id(src_idx) << "][candidate " + << block_id << "]: this node already has candidate in round " << round; + return false; + } + src_round_candidate_[src_idx][round] = block_id; + return true; +} + +void ValidatorSessionImpl::process_broadcast(PublicKeyHash src, td::BufferSlice data, + td::optional expected_id, + bool is_overlay_broadcast) { + // Note: src is not necessarily equal to the sender of this message: + // If requested using get_broadcast_p2p, src is the creator of the block, sender possibly is some other node. auto src_idx = description().get_source_idx(src); auto R = fetch_tl_object(data.clone(), true); if (R.is_error()) { @@ -230,6 +246,12 @@ void ValidatorSessionImpl::process_broadcast(PublicKeyHash src, td::BufferSlice auto block_round = static_cast(candidate->round_); auto block_id = description().candidate_id(src_idx, candidate->root_hash_, file_hash, collated_data_file_hash); + if (expected_id && expected_id.value() != block_id) { + VLOG(VALIDATOR_SESSION_WARNING) << this << "[node " << src << "][broadcast " << sha256_bits256(data.as_slice()) + << "]: id mismatch"; + return; + } + if ((td::int32)block_round < (td::int32)cur_round_ - MAX_PAST_ROUND_BLOCK || block_round >= cur_round_ + MAX_FUTURE_ROUND_BLOCK) { VLOG(VALIDATOR_SESSION_NOTICE) << this << "[node " << src << "][broadcast " << block_id @@ -250,6 +272,10 @@ void ValidatorSessionImpl::process_broadcast(PublicKeyHash src, td::BufferSlice return; } + if (is_overlay_broadcast && !ensure_candidate_unique(src_idx, block_round, block_id)) { + return; + } + blocks_[block_id] = std::move(candidate); VLOG(VALIDATOR_SESSION_WARNING) << this << ": received broadcast " << block_id; @@ -508,7 +534,11 @@ void ValidatorSessionImpl::try_approve_block(const SentBlock *block) { } if (block) { - auto T = td::Timestamp::at(round_started_at_.at() + description().get_delay(block->get_src_idx()) + 2.0); + if (!ensure_candidate_unique(block->get_src_idx(), cur_round_, SentBlock::get_block_id(block))) { + return; + } + auto T = td::Timestamp::at(round_started_at_.at() + description().get_delay(block->get_src_idx()) + + REQUEST_BROADCAST_P2P_DELAY); auto it = blocks_.find(block_id); if (it != blocks_.end()) { @@ -545,16 +575,18 @@ void ValidatorSessionImpl::try_approve_block(const SentBlock *block) { auto id = description().get_source_id(v[td::Random::fast(0, static_cast(v.size() - 1))]); auto src_id = description().get_source_id(block->get_src_idx()); active_requests_.insert(block_id); - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), id, src_id, print_id = print_id(), - hash = block_id, round = cur_round_](td::Result R) { - td::actor::send_closure(SelfId, &ValidatorSessionImpl::end_request, round, hash); - if (R.is_error()) { - VLOG(VALIDATOR_SESSION_WARNING) - << print_id << ": failed to get candidate " << hash << " from " << id << ": " << R.move_as_error(); - } else { - td::actor::send_closure(SelfId, &ValidatorSessionImpl::process_broadcast, src_id, R.move_as_ok()); - } - }); + auto P = td::PromiseCreator::lambda( + [SelfId = actor_id(this), id, src_id, print_id = print_id(), hash = block_id, round = cur_round_, + candidate_id = SentBlock::get_block_id(block)](td::Result R) { + td::actor::send_closure(SelfId, &ValidatorSessionImpl::end_request, round, hash); + if (R.is_error()) { + VLOG(VALIDATOR_SESSION_WARNING) << print_id << ": failed to get candidate " << hash << " from " << id + << ": " << R.move_as_error(); + } else { + td::actor::send_closure(SelfId, &ValidatorSessionImpl::process_broadcast, src_id, R.move_as_ok(), + candidate_id, false); + } + }); get_broadcast_p2p(id, block->get_file_hash(), block->get_collated_data_file_hash(), description().get_source_id(block->get_src_idx()), cur_round_, block->get_root_hash(), @@ -586,10 +618,11 @@ void ValidatorSessionImpl::get_broadcast_p2p(PublicKeyHash node, ValidatorSessio round, create_tl_object(src.tl(), root_hash, file_hash, collated_data_file_hash)); - td::actor::send_closure(catchain_, &catchain::CatChain::send_query_via, node, "download candidate", - std::move(promise), timeout, serialize_tl_object(obj, true), - description().opts().max_block_size + description().opts().max_collated_data_size + 1024, - rldp_); + td::actor::send_closure( + catchain_, &catchain::CatChain::send_query_via, node, "download candidate", std::move(promise), timeout, + serialize_tl_object(obj, true), + description().opts().max_block_size + description().opts().max_collated_data_size + MAX_CANDIDATE_EXTRA_SIZE, + rldp_); } void ValidatorSessionImpl::check_sign_slot() { @@ -745,7 +778,6 @@ void ValidatorSessionImpl::on_new_round(td::uint32 round) { while (cur_round_ < round) { auto block = real_state_->get_committed_block(description(), cur_round_); - //CHECK(block); auto sigs = real_state_->get_committed_block_signatures(description(), cur_round_); CHECK(sigs); auto approve_sigs = real_state_->get_committed_block_approve_signatures(description(), cur_round_); @@ -838,7 +870,8 @@ void ValidatorSessionImpl::on_catchain_started() { auto broadcast = create_tl_object( src.tl(), round, root_hash, std::move(B.data), std::move(B.collated_data)); td::actor::send_closure(SelfId, &ValidatorSessionImpl::process_broadcast, src, - serialize_tl_object(broadcast, true)); + serialize_tl_object(broadcast, true), td::optional(), + false); } }); callback_->get_approved_candidate(description().get_source_public_key(x->get_src_idx()), x->get_root_hash(), @@ -866,6 +899,7 @@ ValidatorSessionImpl::ValidatorSessionImpl(catchain::CatChainSessionId session_i , overlay_manager_(overlays) , allow_unsafe_self_blocks_resync_(allow_unsafe_self_blocks_resync) { description_ = ValidatorSessionDescription::create(std::move(opts), nodes, local_id); + src_round_candidate_.resize(description_->get_total_nodes()); } void ValidatorSessionImpl::start() { diff --git a/validator-session/validator-session.hpp b/validator-session/validator-session.hpp index e82745547..1717c99f0 100644 --- a/validator-session/validator-session.hpp +++ b/validator-session/validator-session.hpp @@ -74,6 +74,8 @@ class ValidatorSessionImpl : public ValidatorSession { td::BufferSlice signature_; std::map> blocks_; + // src_round_candidate_[src_id][round] -> candidate id + std::vector> src_round_candidate_; catchain::CatChainSessionId unique_hash_; @@ -110,7 +112,8 @@ class ValidatorSessionImpl : public ValidatorSession { td::actor::send_closure(id_, &ValidatorSessionImpl::preprocess_block, block); } void process_broadcast(const PublicKeyHash &src, td::BufferSlice data) override { - td::actor::send_closure(id_, &ValidatorSessionImpl::process_broadcast, src, std::move(data)); + td::actor::send_closure(id_, &ValidatorSessionImpl::process_broadcast, src, std::move(data), + td::optional(), true); } void process_message(const PublicKeyHash &src, td::BufferSlice data) override { td::actor::send_closure(id_, &ValidatorSessionImpl::process_message, src, std::move(data)); @@ -174,7 +177,9 @@ class ValidatorSessionImpl : public ValidatorSession { void process_blocks(std::vector blocks); void finished_processing(); void preprocess_block(catchain::CatChainBlock *block); - void process_broadcast(PublicKeyHash src, td::BufferSlice data); + bool ensure_candidate_unique(td::uint32 src_idx, td::uint32 round, ValidatorSessionCandidateId block_id); + void process_broadcast(PublicKeyHash src, td::BufferSlice data, td::optional expected_id, + bool is_overlay_broadcast); void process_message(PublicKeyHash src, td::BufferSlice data); void process_query(PublicKeyHash src, td::BufferSlice data, td::Promise promise); @@ -206,6 +211,8 @@ class ValidatorSessionImpl : public ValidatorSession { static const size_t MAX_REJECT_REASON_SIZE = 1024; static const td::int32 MAX_FUTURE_ROUND_BLOCK = 100; static const td::int32 MAX_PAST_ROUND_BLOCK = 20; + constexpr static const double REQUEST_BROADCAST_P2P_DELAY = 2.0; + static const td::uint32 MAX_CANDIDATE_EXTRA_SIZE = 1024; }; } // namespace validatorsession From 583178ccb165db42a69556aa58cd12d0f19ef6a6 Mon Sep 17 00:00:00 2001 From: EmelyanenkoK Date: Mon, 15 May 2023 15:31:42 +0300 Subject: [PATCH 18/20] FunC: enable asserts and fix try/catch stack corruption (#699) * FunC: enable asserts in Release * FunC: Fix analyzing infinite loops * FunC: Allow catch with one tensor argument * FunC: Fix try/catch stack corruption --------- Co-authored-by: SpyCheese --- crypto/func/abscode.cpp | 3 + crypto/func/analyzer.cpp | 26 +++--- crypto/func/auto-tests/tests/try-func.fc | 109 +++++++++++++++++++++++ crypto/func/builtins.cpp | 54 +++++------ crypto/func/codegen.cpp | 93 +++++++++++-------- crypto/func/func.cpp | 9 +- crypto/func/func.h | 34 +++++-- crypto/func/gen-abscode.cpp | 18 ++-- crypto/func/optimize.cpp | 28 +++--- crypto/func/parse-func.cpp | 9 +- crypto/func/stack-transform.cpp | 2 +- crypto/func/unify-types.cpp | 20 ++--- 12 files changed, 276 insertions(+), 129 deletions(-) create mode 100644 crypto/func/auto-tests/tests/try-func.fc diff --git a/crypto/func/abscode.cpp b/crypto/func/abscode.cpp index 933edc3d9..f1ffcfa49 100644 --- a/crypto/func/abscode.cpp +++ b/crypto/func/abscode.cpp @@ -38,6 +38,9 @@ TmpVar::TmpVar(var_idx_t _idx, int _cls, TypeExpr* _type, SymDef* sym, const Src if (!_type) { v_type = TypeExpr::new_hole(); } + if (cls == _Named) { + undefined = true; + } } void TmpVar::set_location(const SrcLocation& loc) { diff --git a/crypto/func/analyzer.cpp b/crypto/func/analyzer.cpp index d59230e87..ec6931af0 100644 --- a/crypto/func/analyzer.cpp +++ b/crypto/func/analyzer.cpp @@ -42,14 +42,14 @@ int CodeBlob::split_vars(bool strict) { } std::vector comp_types; int k = var.v_type->extract_components(comp_types); - assert(k <= 254 && n <= 0x7fff00); - assert((unsigned)k == comp_types.size()); + func_assert(k <= 254 && n <= 0x7fff00); + func_assert((unsigned)k == comp_types.size()); if (k != 1) { var.coord = ~((n << 8) + k); for (int i = 0; i < k; i++) { auto v = create_var(vars[j].cls, comp_types[i], 0, vars[j].where.get()); - assert(v == n + i); - assert(vars[v].idx == v); + func_assert(v == n + i); + func_assert(vars[v].idx == v); vars[v].name = vars[j].name; vars[v].coord = ((int)j << 8) + i + 1; } @@ -75,9 +75,9 @@ bool CodeBlob::compute_used_code_vars() { } bool CodeBlob::compute_used_code_vars(std::unique_ptr& ops_ptr, const VarDescrList& var_info, bool edit) const { - assert(ops_ptr); + func_assert(ops_ptr); if (!ops_ptr->next) { - assert(ops_ptr->cl == Op::_Nop); + func_assert(ops_ptr->cl == Op::_Nop); return ops_ptr->set_var_info(var_info); } return compute_used_code_vars(ops_ptr->next, var_info, edit) | ops_ptr->compute_used_vars(*this, edit); @@ -346,7 +346,7 @@ bool Op::std_compute_used_vars(bool disabled) { } bool Op::compute_used_vars(const CodeBlob& code, bool edit) { - assert(next); + func_assert(next); const VarDescrList& next_var_info = next->var_info; if (cl == _Nop) { return set_var_info_except(next_var_info, left); @@ -379,7 +379,7 @@ bool Op::compute_used_vars(const CodeBlob& code, bool edit) { case _Let: { // left = right std::size_t cnt = next_var_info.count_used(left); - assert(left.size() == right.size()); + func_assert(left.size() == right.size()); auto l_it = left.cbegin(), r_it = right.cbegin(); VarDescrList new_var_info{next_var_info}; new_var_info -= left; @@ -500,7 +500,7 @@ bool Op::compute_used_vars(const CodeBlob& code, bool edit) { } changes = (new_var_info.size() == n); } while (changes <= edit); - assert(left.size() == 1); + func_assert(left.size() == 1); bool last = new_var_info.count_used(left) == 0; new_var_info += left; if (last) { @@ -655,7 +655,7 @@ bool prune_unreachable(std::unique_ptr& ops) { ops = std::move(op.block0); return false; } - reach = true; + reach = (op.cl != Op::_Again); break; } case Op::_TryCatch: { @@ -684,7 +684,7 @@ void CodeBlob::prune_unreachable_code() { void CodeBlob::fwd_analyze() { VarDescrList values; - assert(ops && ops->cl == Op::_Import); + func_assert(ops && ops->cl == Op::_Import); for (var_idx_t i : ops->left) { values += i; if (vars[i].v_type->is_int()) { @@ -765,7 +765,7 @@ VarDescrList Op::fwd_analyze(VarDescrList values) { break; case _Let: { std::vector old_val; - assert(left.size() == right.size()); + func_assert(left.size() == right.size()); for (std::size_t i = 0; i < right.size(); i++) { const VarDescr* ov = values[right[i]]; if (!ov && verbosity >= 5) { @@ -780,7 +780,7 @@ VarDescrList Op::fwd_analyze(VarDescrList values) { } std::cerr << std::endl; } - // assert(ov); + // func_assert(ov); if (ov) { old_val.push_back(*ov); } else { diff --git a/crypto/func/auto-tests/tests/try-func.fc b/crypto/func/auto-tests/tests/try-func.fc new file mode 100644 index 000000000..5678965ef --- /dev/null +++ b/crypto/func/auto-tests/tests/try-func.fc @@ -0,0 +1,109 @@ +int foo(int x) method_id(11) { + try { + if (x == 7) { + throw(44); + } + return x; + } catch (_, _) { + return 2; + } +} + +int foo_inline(int x) inline method_id(12) { + try { + if (x == 7) { + throw(44); + } + return x; + } catch (_, _) { + return 2; + } +} + +int foo_inlineref(int x) inline_ref method_id(13) { + try { + if (x == 7) { + throw(44); + } + return x; + } catch (_, _) { + return 2; + } +} + +int test(int x, int y, int z) method_id(1) { + y = foo(y); + return x * 100 + y * 10 + z; +} + +int test_inline(int x, int y, int z) method_id(2) { + y = foo_inline(y); + return x * 100 + y * 10 + z; +} + +int test_inlineref(int x, int y, int z) method_id(3) { + y = foo_inlineref(y); + return x * 100 + y * 10 + z; +} + +int foo_inline_big( + int x1, int x2, int x3, int x4, int x5, int x6, int x7, int x8, int x9, int x10, + int x11, int x12, int x13, int x14, int x15, int x16, int x17, int x18, int x19, int x20 +) inline method_id(14) { + try { + if (x1 == 7) { + throw(44); + } + return x1 + x2 + x3 + x4 + x5 + x6 + x7 + x8 + x9 + x10 + x11 + x12 + x13 + x14 + x15 + x16 + x17 + x18 + x19 + x20; + } catch (_, _) { + return 1; + } +} + +int test_inline_big(int x, int y, int z) method_id(4) { + y = foo_inline_big( + y, y + 1, y + 2, y + 3, y + 4, y + 5, y + 6, y + 7, y + 8, y + 9, + y + 10, y + 11, y + 12, y + 13, y + 14, y + 15, y + 16, y + 17, y + 18, y + 19); + return x * 1000000 + y * 1000 + z; +} + +int foo_big( + int x1, int x2, int x3, int x4, int x5, int x6, int x7, int x8, int x9, int x10, + int x11, int x12, int x13, int x14, int x15, int x16, int x17, int x18, int x19, int x20 +) method_id(15) { + try { + if (x1 == 7) { + throw(44); + } + return x1 + x2 + x3 + x4 + x5 + x6 + x7 + x8 + x9 + x10 + x11 + x12 + x13 + x14 + x15 + x16 + x17 + x18 + x19 + x20; + } catch (_, _) { + return 1; + } +} + +int test_big(int x, int y, int z) method_id(5) { + y = foo_big( + y, y + 1, y + 2, y + 3, y + 4, y + 5, y + 6, y + 7, y + 8, y + 9, + y + 10, y + 11, y + 12, y + 13, y + 14, y + 15, y + 16, y + 17, y + 18, y + 19); + return x * 1000000 + y * 1000 + z; +} + +() main() { +} + +{- + method_id | in | out +TESTCASE | 1 | 1 2 3 | 123 +TESTCASE | 1 | 3 8 9 | 389 +TESTCASE | 1 | 3 7 9 | 329 +TESTCASE | 2 | 1 2 3 | 123 +TESTCASE | 2 | 3 8 9 | 389 +TESTCASE | 2 | 3 7 9 | 329 +TESTCASE | 3 | 1 2 3 | 123 +TESTCASE | 3 | 3 8 9 | 389 +TESTCASE | 3 | 3 7 9 | 329 +TESTCASE | 4 | 4 8 9 | 4350009 +TESTCASE | 4 | 4 7 9 | 4001009 +TESTCASE | 5 | 4 8 9 | 4350009 +TESTCASE | 5 | 4 7 9 | 4001009 +-} diff --git a/crypto/func/builtins.cpp b/crypto/func/builtins.cpp index 296910347..9de673fd7 100644 --- a/crypto/func/builtins.cpp +++ b/crypto/func/builtins.cpp @@ -429,7 +429,7 @@ AsmOp push_const(td::RefInt256 x) { } AsmOp compile_add(std::vector& res, std::vector& args, const SrcLocation& where) { - assert(res.size() == 1 && args.size() == 2); + func_assert(res.size() == 1 && args.size() == 2); VarDescr &r = res[0], &x = args[0], &y = args[1]; if (x.is_int_const() && y.is_int_const()) { r.set_const(x.int_const + y.int_const); @@ -471,7 +471,7 @@ AsmOp compile_add(std::vector& res, std::vector& args, const } AsmOp compile_sub(std::vector& res, std::vector& args, const SrcLocation& where) { - assert(res.size() == 1 && args.size() == 2); + func_assert(res.size() == 1 && args.size() == 2); VarDescr &r = res[0], &x = args[0], &y = args[1]; if (x.is_int_const() && y.is_int_const()) { r.set_const(x.int_const - y.int_const); @@ -504,7 +504,7 @@ AsmOp compile_sub(std::vector& res, std::vector& args, const } AsmOp compile_negate(std::vector& res, std::vector& args, const SrcLocation& where) { - assert(res.size() == 1 && args.size() == 1); + func_assert(res.size() == 1 && args.size() == 1); VarDescr &r = res[0], &x = args[0]; if (x.is_int_const()) { r.set_const(-x.int_const); @@ -519,7 +519,7 @@ AsmOp compile_negate(std::vector& res, std::vector& args, co } AsmOp compile_and(std::vector& res, std::vector& args, const SrcLocation& where) { - assert(res.size() == 1 && args.size() == 2); + func_assert(res.size() == 1 && args.size() == 2); VarDescr &r = res[0], &x = args[0], &y = args[1]; if (x.is_int_const() && y.is_int_const()) { r.set_const(x.int_const & y.int_const); @@ -532,7 +532,7 @@ AsmOp compile_and(std::vector& res, std::vector& args, const } AsmOp compile_or(std::vector& res, std::vector& args, const SrcLocation& where) { - assert(res.size() == 1 && args.size() == 2); + func_assert(res.size() == 1 && args.size() == 2); VarDescr &r = res[0], &x = args[0], &y = args[1]; if (x.is_int_const() && y.is_int_const()) { r.set_const(x.int_const | y.int_const); @@ -545,7 +545,7 @@ AsmOp compile_or(std::vector& res, std::vector& args, const } AsmOp compile_xor(std::vector& res, std::vector& args, const SrcLocation& where) { - assert(res.size() == 1 && args.size() == 2); + func_assert(res.size() == 1 && args.size() == 2); VarDescr &r = res[0], &x = args[0], &y = args[1]; if (x.is_int_const() && y.is_int_const()) { r.set_const(x.int_const ^ y.int_const); @@ -558,7 +558,7 @@ AsmOp compile_xor(std::vector& res, std::vector& args, const } AsmOp compile_not(std::vector& res, std::vector& args, const SrcLocation& where) { - assert(res.size() == 1 && args.size() == 1); + func_assert(res.size() == 1 && args.size() == 1); VarDescr &r = res[0], &x = args[0]; if (x.is_int_const()) { r.set_const(~x.int_const); @@ -638,12 +638,12 @@ AsmOp compile_mul_internal(VarDescr& r, VarDescr& x, VarDescr& y, const SrcLocat } AsmOp compile_mul(std::vector& res, std::vector& args, const SrcLocation& where) { - assert(res.size() == 1 && args.size() == 2); + func_assert(res.size() == 1 && args.size() == 2); return compile_mul_internal(res[0], args[0], args[1], where); } AsmOp compile_lshift(std::vector& res, std::vector& args, const SrcLocation& where) { - assert(res.size() == 1 && args.size() == 2); + func_assert(res.size() == 1 && args.size() == 2); VarDescr &r = res[0], &x = args[0], &y = args[1]; if (y.is_int_const()) { auto yv = y.int_const->to_long(); @@ -686,7 +686,7 @@ AsmOp compile_lshift(std::vector& res, std::vector& args, co AsmOp compile_rshift(std::vector& res, std::vector& args, const SrcLocation& where, int round_mode) { - assert(res.size() == 1 && args.size() == 2); + func_assert(res.size() == 1 && args.size() == 2); VarDescr &r = res[0], &x = args[0], &y = args[1]; if (y.is_int_const()) { auto yv = y.int_const->to_long(); @@ -755,13 +755,13 @@ AsmOp compile_div_internal(VarDescr& r, VarDescr& x, VarDescr& y, const SrcLocat } AsmOp compile_div(std::vector& res, std::vector& args, const SrcLocation& where, int round_mode) { - assert(res.size() == 1 && args.size() == 2); + func_assert(res.size() == 1 && args.size() == 2); return compile_div_internal(res[0], args[0], args[1], where, round_mode); } AsmOp compile_mod(std::vector& res, std::vector& args, const src::SrcLocation& where, int round_mode) { - assert(res.size() == 1 && args.size() == 2); + func_assert(res.size() == 1 && args.size() == 2); VarDescr &r = res[0], &x = args[0], &y = args[1]; if (x.is_int_const() && y.is_int_const()) { r.set_const(mod(x.int_const, y.int_const, round_mode)); @@ -802,7 +802,7 @@ AsmOp compile_mod(std::vector& res, std::vector& args, const AsmOp compile_muldiv(std::vector& res, std::vector& args, const SrcLocation& where, int round_mode) { - assert(res.size() == 1 && args.size() == 3); + func_assert(res.size() == 1 && args.size() == 3); VarDescr &r = res[0], &x = args[0], &y = args[1], &z = args[2]; if (x.is_int_const() && y.is_int_const() && z.is_int_const()) { r.set_const(muldiv(x.int_const, y.int_const, z.int_const, round_mode)); @@ -923,8 +923,8 @@ int compute_compare(const VarDescr& x, const VarDescr& y, int mode) { } AsmOp compile_cmp_int(std::vector& res, std::vector& args, int mode) { - assert(mode >= 1 && mode <= 7); - assert(res.size() == 1 && args.size() == 2); + func_assert(mode >= 1 && mode <= 7); + func_assert(res.size() == 1 && args.size() == 2); VarDescr &r = res[0], &x = args[0], &y = args[1]; if (x.is_int_const() && y.is_int_const()) { int v = compute_compare(x.int_const, y.int_const, mode); @@ -935,7 +935,7 @@ AsmOp compile_cmp_int(std::vector& res, std::vector& args, i } int v = compute_compare(x, y, mode); // std::cerr << "compute_compare(" << x << ", " << y << ", " << mode << ") = " << v << std::endl; - assert(v); + func_assert(v); if (!(v & (v - 1))) { r.set_const(v - (v >> 2) - 2); x.unused(); @@ -971,7 +971,7 @@ AsmOp compile_cmp_int(std::vector& res, std::vector& args, i } AsmOp compile_throw(std::vector& res, std::vector& args, const SrcLocation&) { - assert(res.empty() && args.size() == 1); + func_assert(res.empty() && args.size() == 1); VarDescr& x = args[0]; if (x.is_int_const() && x.int_const->unsigned_fits_bits(11)) { x.unused(); @@ -982,7 +982,7 @@ AsmOp compile_throw(std::vector& res, std::vector& args, con } AsmOp compile_cond_throw(std::vector& res, std::vector& args, bool mode) { - assert(res.empty() && args.size() == 2); + func_assert(res.empty() && args.size() == 2); VarDescr &x = args[0], &y = args[1]; std::string suff = (mode ? "IF" : "IFNOT"); bool skip_cond = false; @@ -1003,7 +1003,7 @@ AsmOp compile_cond_throw(std::vector& res, std::vector& args } AsmOp compile_throw_arg(std::vector& res, std::vector& args, const SrcLocation&) { - assert(res.empty() && args.size() == 2); + func_assert(res.empty() && args.size() == 2); VarDescr &x = args[1]; if (x.is_int_const() && x.int_const->unsigned_fits_bits(11)) { x.unused(); @@ -1014,7 +1014,7 @@ AsmOp compile_throw_arg(std::vector& res, std::vector& args, } AsmOp compile_cond_throw_arg(std::vector& res, std::vector& args, bool mode) { - assert(res.empty() && args.size() == 3); + func_assert(res.empty() && args.size() == 3); VarDescr &x = args[1], &y = args[2]; std::string suff = (mode ? "IF" : "IFNOT"); bool skip_cond = false; @@ -1035,7 +1035,7 @@ AsmOp compile_cond_throw_arg(std::vector& res, std::vector& } AsmOp compile_bool_const(std::vector& res, std::vector& args, bool val) { - assert(res.size() == 1 && args.empty()); + func_assert(res.size() == 1 && args.empty()); VarDescr& r = res[0]; r.set_const(val ? -1 : 0); return AsmOp::Const(val ? "TRUE" : "FALSE"); @@ -1046,7 +1046,7 @@ AsmOp compile_bool_const(std::vector& res, std::vector& args // int preload_int(slice s, int len) asm "PLDIX"; // int preload_uint(slice s, int len) asm "PLDUX"; AsmOp compile_fetch_int(std::vector& res, std::vector& args, bool fetch, bool sgnd) { - assert(args.size() == 2 && res.size() == 1 + (unsigned)fetch); + func_assert(args.size() == 2 && res.size() == 1 + (unsigned)fetch); auto &y = args[1], &r = res.back(); r.val = (sgnd ? VarDescr::FiniteInt : VarDescr::FiniteUInt); int v = -1; @@ -1069,7 +1069,7 @@ AsmOp compile_fetch_int(std::vector& res, std::vector& args, // builder store_uint(builder b, int x, int len) asm(x b len) "STUX"; // builder store_int(builder b, int x, int len) asm(x b len) "STIX"; AsmOp compile_store_int(std::vector& res, std::vector& args, bool sgnd) { - assert(args.size() == 3 && res.size() == 1); + func_assert(args.size() == 3 && res.size() == 1); auto& z = args[2]; if (z.is_int_const() && z.int_const > 0 && z.int_const <= 256) { z.unused(); @@ -1079,7 +1079,7 @@ AsmOp compile_store_int(std::vector& res, std::vector& args, } AsmOp compile_fetch_slice(std::vector& res, std::vector& args, bool fetch) { - assert(args.size() == 2 && res.size() == 1 + (unsigned)fetch); + func_assert(args.size() == 2 && res.size() == 1 + (unsigned)fetch); auto& y = args[1]; int v = -1; if (y.is_int_const() && y.int_const > 0 && y.int_const <= 256) { @@ -1094,7 +1094,7 @@ AsmOp compile_fetch_slice(std::vector& res, std::vector& arg // _at(tuple t, int index) asm "INDEXVAR"; AsmOp compile_tuple_at(std::vector& res, std::vector& args, const SrcLocation&) { - assert(args.size() == 2 && res.size() == 1); + func_assert(args.size() == 2 && res.size() == 1); auto& y = args[1]; if (y.is_int_const() && y.int_const >= 0 && y.int_const < 16) { y.unused(); @@ -1105,7 +1105,7 @@ AsmOp compile_tuple_at(std::vector& res, std::vector& args, // int null?(X arg) AsmOp compile_is_null(std::vector& res, std::vector& args, const SrcLocation&) { - assert(args.size() == 1 && res.size() == 1); + func_assert(args.size() == 1 && res.size() == 1); auto &x = args[0], &r = res[0]; if (x.always_null() || x.always_not_null()) { x.unused(); @@ -1118,7 +1118,7 @@ AsmOp compile_is_null(std::vector& res, std::vector& args, c bool compile_run_method(AsmOpList& code, std::vector& res, std::vector& args, int n, bool has_value) { - assert(args.size() == (unsigned)n + 1 && res.size() == (unsigned)has_value); + func_assert(args.size() == (unsigned)n + 1 && res.size() == (unsigned)has_value); auto& x = args[0]; if (x.is_int_const() && x.int_const->unsigned_fits_bits(14)) { x.unused(); diff --git a/crypto/func/codegen.cpp b/crypto/func/codegen.cpp index 872909655..de45c8418 100644 --- a/crypto/func/codegen.cpp +++ b/crypto/func/codegen.cpp @@ -165,7 +165,7 @@ void Stack::push_new_const(var_idx_t idx, const_idx_t cidx) { void Stack::assign_var(var_idx_t new_idx, var_idx_t old_idx) { int i = find(old_idx); - assert(i >= 0 && "variable not found in stack"); + func_assert(i >= 0 && "variable not found in stack"); if (new_idx != old_idx) { at(i).first = new_idx; modified(); @@ -174,10 +174,10 @@ void Stack::assign_var(var_idx_t new_idx, var_idx_t old_idx) { void Stack::do_copy_var(var_idx_t new_idx, var_idx_t old_idx) { int i = find(old_idx); - assert(i >= 0 && "variable not found in stack"); + func_assert(i >= 0 && "variable not found in stack"); if (find(old_idx, i + 1) < 0) { issue_push(i); - assert(at(0).first == old_idx); + func_assert(at(0).first == old_idx); } assign_var(new_idx, old_idx); } @@ -199,21 +199,21 @@ void Stack::enforce_state(const StackLayout& req_stack) { j = 0; } issue_xchg(j, depth() - i - 1); - assert(s[i].first == x); + func_assert(s[i].first == x); } while (depth() > k) { issue_pop(0); } - assert(depth() == k); + func_assert(depth() == k); for (int i = 0; i < k; i++) { - assert(s[i].first == req_stack[i]); + func_assert(s[i].first == req_stack[i]); } } void Stack::merge_const(const Stack& req_stack) { - assert(s.size() == req_stack.s.size()); + func_assert(s.size() == req_stack.s.size()); for (std::size_t i = 0; i < s.size(); i++) { - assert(s[i].first == req_stack.s[i].first); + func_assert(s[i].first == req_stack.s[i].first); if (s[i].second != req_stack.s[i].second) { s[i].second = not_const; } @@ -251,15 +251,15 @@ void Stack::rearrange_top(const StackLayout& top, std::vector last) { if (last[i]) { // rearrange x to be at s(ss-1) issue_xchg(--ss, j); - assert(get(ss).first == x); + func_assert(get(ss).first == x); } else { // create a new copy of x issue_push(j); issue_xchg(0, ss); - assert(get(ss).first == x); + func_assert(get(ss).first == x); } } - assert(!ss); + func_assert(!ss); } void Stack::rearrange_top(var_idx_t top, bool last) { @@ -269,7 +269,7 @@ void Stack::rearrange_top(var_idx_t top, bool last) { } else { issue_push(i); } - assert(get(0).first == top); + func_assert(get(0).first == top); } bool Op::generate_code_step(Stack& stack) { @@ -300,7 +300,7 @@ bool Op::generate_code_step(Stack& stack) { stack.o << push_const(int_const); stack.push_new_const(left[0], cidx); } else { - assert(stack.at(i).second == cidx); + func_assert(stack.at(i).second == cidx); stack.do_copy_var(left[0], stack[i]); } return true; @@ -329,7 +329,7 @@ bool Op::generate_code_step(Stack& stack) { std::string name = sym::symbols.get_name(fun_ref->sym_idx); stack.o << AsmOp::Custom(name + " GETGLOB", 0, 1); if (left.size() != 1) { - assert(left.size() <= 15); + func_assert(left.size() <= 15); stack.o << AsmOp::UnTuple((int)left.size()); } for (auto i : left) { @@ -337,7 +337,7 @@ bool Op::generate_code_step(Stack& stack) { } return true; } else { - assert(left.size() == 1); + func_assert(left.size() == 1); auto p = next->var_info[left[0]]; if (!p || p->is_unused() || disabled()) { return true; @@ -349,10 +349,10 @@ bool Op::generate_code_step(Stack& stack) { // TODO: create and compile a true lambda instead of this (so that arg_order and ret_order would work correctly) std::vector args0, res; TypeExpr::remove_indirect(func->sym_type); - assert(func->get_type()->is_map()); + func_assert(func->get_type()->is_map()); auto wr = func->get_type()->args.at(0)->get_width(); auto wl = func->get_type()->args.at(1)->get_width(); - assert(wl >= 0 && wr >= 0); + func_assert(wl >= 0 && wr >= 0); for (int i = 0; i < wl; i++) { res.emplace_back(0); } @@ -370,7 +370,7 @@ bool Op::generate_code_step(Stack& stack) { return true; } case _Let: { - assert(left.size() == right.size()); + func_assert(left.size() == right.size()); int i = 0; std::vector active; active.reserve(left.size()); @@ -420,13 +420,13 @@ bool Op::generate_code_step(Stack& stack) { stack.rearrange_top(right, std::move(last)); stack.opt_show(); int k = (int)stack.depth() - (int)right.size(); - assert(k >= 0); + func_assert(k >= 0); if (cl == _Tuple) { stack.o << AsmOp::Tuple((int)right.size()); - assert(left.size() == 1); + func_assert(left.size() == 1); } else { stack.o << AsmOp::UnTuple((int)left.size()); - assert(right.size() == 1); + func_assert(right.size() == 1); } stack.s.resize(k); for (int i = 0; i < (int)left.size(); i++) { @@ -442,16 +442,16 @@ bool Op::generate_code_step(Stack& stack) { SymValFunc* func = (fun_ref ? dynamic_cast(fun_ref->value) : nullptr); auto arg_order = (func ? func->get_arg_order() : nullptr); auto ret_order = (func ? func->get_ret_order() : nullptr); - assert(!arg_order || arg_order->size() == right.size()); - assert(!ret_order || ret_order->size() == left.size()); + func_assert(!arg_order || arg_order->size() == right.size()); + func_assert(!ret_order || ret_order->size() == left.size()); std::vector right1; if (args.size()) { - assert(args.size() == right.size()); + func_assert(args.size() == right.size()); for (int i = 0; i < (int)right.size(); i++) { int j = arg_order ? arg_order->at(i) : i; const VarDescr& arg = args.at(j); if (!arg.is_unused()) { - assert(var_info[arg.idx] && !var_info[arg.idx]->is_unused()); + func_assert(var_info[arg.idx] && !var_info[arg.idx]->is_unused()); right1.push_back(arg.idx); } } @@ -469,17 +469,25 @@ bool Op::generate_code_step(Stack& stack) { stack.rearrange_top(right1, std::move(last)); stack.opt_show(); int k = (int)stack.depth() - (int)right1.size(); - assert(k >= 0); + func_assert(k >= 0); for (int i = 0; i < (int)right1.size(); i++) { if (stack.s[k + i].first != right1[i]) { std::cerr << stack.o; } - assert(stack.s[k + i].first == right1[i]); + func_assert(stack.s[k + i].first == right1[i]); } + auto exec_callxargs = [&](int args, int ret) { + if (args <= 15 && ret <= 15) { + stack.o << exec_arg2_op("CALLXARGS", args, ret, args + 1, ret); + } else { + func_assert(args <= 254 && ret <= 254); + stack.o << AsmOp::Const(PSTRING() << args << " PUSHINT"); + stack.o << AsmOp::Const(PSTRING() << ret << " PUSHINT"); + stack.o << AsmOp::Custom("CALLXVARARGS", args + 3, ret); + } + }; if (cl == _CallInd) { - // TODO: replace with exec_arg2_op() - stack.o << exec_arg2_op("CALLXARGS", (int)right.size() - 1, (int)left.size(), (int)right.size(), - (int)left.size()); + exec_callxargs((int)right.size() - 1, (int)left.size()); } else { auto func = dynamic_cast(fun_ref->value); if (func) { @@ -493,8 +501,14 @@ bool Op::generate_code_step(Stack& stack) { auto fv = dynamic_cast(fun_ref->value); std::string name = sym::symbols.get_name(fun_ref->sym_idx); bool is_inline = (fv && (fv->flags & 3)); - stack.o << AsmOp::Custom(name + (is_inline ? " INLINECALLDICT" : " CALLDICT"), (int)right.size(), - (int)left.size()); + if (is_inline) { + stack.o << AsmOp::Custom(name + " INLINECALLDICT", (int)right.size(), (int)left.size()); + } else if (fv && fv->code && fv->code->require_callxargs) { + stack.o << AsmOp::Custom(name + (" PREPAREDICT"), 0, 2); + exec_callxargs((int)right.size() + 1, (int)left.size()); + } else { + stack.o << AsmOp::Custom(name + " CALLDICT", (int)right.size(), (int)left.size()); + } } } stack.s.resize(k); @@ -505,7 +519,7 @@ bool Op::generate_code_step(Stack& stack) { return true; } case _SetGlob: { - assert(fun_ref && dynamic_cast(fun_ref->value)); + func_assert(fun_ref && dynamic_cast(fun_ref->value)); std::vector last; for (var_idx_t x : right) { last.push_back(var_info[x] && var_info[x]->is_last()); @@ -513,12 +527,12 @@ bool Op::generate_code_step(Stack& stack) { stack.rearrange_top(right, std::move(last)); stack.opt_show(); int k = (int)stack.depth() - (int)right.size(); - assert(k >= 0); + func_assert(k >= 0); for (int i = 0; i < (int)right.size(); i++) { if (stack.s[k + i].first != right[i]) { std::cerr << stack.o; } - assert(stack.s[k + i].first == right[i]); + func_assert(stack.s[k + i].first == right[i]); } if (right.size() > 1) { stack.o << AsmOp::Tuple((int)right.size()); @@ -539,7 +553,7 @@ bool Op::generate_code_step(Stack& stack) { } var_idx_t x = left[0]; stack.rearrange_top(x, var_info[x] && var_info[x]->is_last()); - assert(stack[0] == x); + func_assert(stack[0] == x); stack.opt_show(); stack.s.pop_back(); stack.modified(); @@ -651,7 +665,7 @@ bool Op::generate_code_step(Stack& stack) { var_idx_t x = left[0]; //stack.drop_vars_except(block0->var_info, x); stack.rearrange_top(x, var_info[x] && var_info[x]->is_last()); - assert(stack[0] == x); + func_assert(stack[0] == x); stack.opt_show(); stack.s.pop_back(); stack.modified(); @@ -877,12 +891,13 @@ void Op::generate_code_all(Stack& stack) { void CodeBlob::generate_code(AsmOpList& out, int mode) { Stack stack{out, mode}; - assert(ops && ops->cl == Op::_Import); + func_assert(ops && ops->cl == Op::_Import); + auto args = (int)ops->left.size(); for (var_idx_t x : ops->left) { stack.push_new_var(x); } ops->generate_code_all(stack); - stack.apply_wrappers(); + stack.apply_wrappers(require_callxargs && (mode & Stack::_InlineAny) ? args : -1); if (!(mode & Stack::_DisableOpt)) { optimize_code(out); } diff --git a/crypto/func/func.cpp b/crypto/func/func.cpp index 87bf23e98..39648c05a 100644 --- a/crypto/func/func.cpp +++ b/crypto/func/func.cpp @@ -72,7 +72,7 @@ td::Result fs_read_callback(ReadCallback::Kind kind, const char* qu void generate_output_func(SymDef* func_sym, std::ostream &outs, std::ostream &errs) { SymValCodeFunc* func_val = dynamic_cast(func_sym->value); - assert(func_val); + func_assert(func_val); std::string name = sym::symbols.get_name(func_sym->sym_idx); if (verbosity >= 2) { errs << "\n\n=========================\nfunction " << name << " : " << func_val->get_type() << std::endl; @@ -145,6 +145,9 @@ void generate_output_func(SymDef* func_sym, std::ostream &outs, std::ostream &er if (fv && (fv->flags & 1) && code.ops->noreturn()) { mode |= Stack::_InlineFunc; } + if (fv && (fv->flags & 3)) { + mode |= Stack::_InlineAny; + } code.generate_code(outs, mode, indent + 1); outs << std::string(indent * 2, ' ') << "}>\n"; if (verbosity >= 2) { @@ -163,7 +166,7 @@ int generate_output(std::ostream &outs, std::ostream &errs) { } for (SymDef* func_sym : glob_func) { SymValCodeFunc* func_val = dynamic_cast(func_sym->value); - assert(func_val); + func_assert(func_val); std::string name = sym::symbols.get_name(func_sym->sym_idx); outs << std::string(indent * 2, ' '); if (func_val->method_id.is_null()) { @@ -173,7 +176,7 @@ int generate_output(std::ostream &outs, std::ostream &errs) { } } for (SymDef* gvar_sym : glob_vars) { - assert(dynamic_cast(gvar_sym->value)); + func_assert(dynamic_cast(gvar_sym->value)); std::string name = sym::symbols.get_name(gvar_sym->sym_idx); outs << std::string(indent * 2, ' ') << "DECLGLOBVAR " << name << "\n"; } diff --git a/crypto/func/func.h b/crypto/func/func.h index 33f8c86f0..2b95bcbc5 100644 --- a/crypto/func/func.h +++ b/crypto/func/func.h @@ -33,6 +33,10 @@ #include "parser/symtable.h" #include "td/utils/Status.h" +#define func_assert(expr) \ + (bool(expr) ? void(0) \ + : throw src::Fatal(PSTRING() << "Assertion failed at " << __FILE__ << ":" << __LINE__ << ": " << #expr)) + namespace funC { extern int verbosity; @@ -310,6 +314,7 @@ struct TmpVar { int coord; std::unique_ptr where; std::vector> on_modification; + bool undefined = false; TmpVar(var_idx_t _idx, int _cls, TypeExpr* _type = 0, SymDef* sym = 0, const SrcLocation* loc = 0); void show(std::ostream& os, int omit_idx = 0) const; void dump(std::ostream& os) const; @@ -694,6 +699,7 @@ struct CodeBlob { std::unique_ptr* cur_ops; std::stack*> cur_ops_stack; int flags = 0; + bool require_callxargs = false; CodeBlob(TypeExpr* ret = nullptr) : var_cnt(0), in_var_cnt(0), op_cnt(0), ret_type(ret), cur_ops(&ops) { } template @@ -1177,7 +1183,7 @@ struct AsmOpList { } template AsmOpList& add(Args&&... args) { - list_.emplace_back(std::forward(args)...); + append(AsmOp(std::forward(args)...)); adjust_last(); return *this; } @@ -1564,8 +1570,8 @@ struct Stack { AsmOpList& o; enum { _StkCmt = 1, _CptStkCmt = 2, _DisableOpt = 4, _DisableOut = 128, _Shown = 256, - _InlineFunc = 512, _NeedRetAlt = 1024, - _ModeSave = _InlineFunc | _NeedRetAlt, + _InlineFunc = 512, _NeedRetAlt = 1024, _InlineAny = 2048, + _ModeSave = _InlineFunc | _NeedRetAlt | _InlineAny, _Garbage = -0x10000 }; int mode; @@ -1612,7 +1618,7 @@ struct Stack { if (i > 255) { throw src::Fatal{"Too deep stack"}; } - assert(i >= 0 && i < depth() && "invalid stack reference"); + func_assert(i >= 0 && i < depth() && "invalid stack reference"); } void modified() { mode &= ~_Shown; @@ -1643,14 +1649,24 @@ struct Stack { bool operator==(const Stack& y) const & { return s == y.s; } - void apply_wrappers() { + void apply_wrappers(int callxargs_count) { + bool is_inline = mode & _InlineFunc; if (o.retalt_) { o.insert(0, "SAMEALTSAVE"); o.insert(0, "c2 SAVE"); - if (mode & _InlineFunc) { - o.indent_all(); - o.insert(0, "CONT:<{"); - o << "}>"; + } + if (callxargs_count != -1 || (is_inline && o.retalt_)) { + o.indent_all(); + o.insert(0, "CONT:<{"); + o << "}>"; + if (callxargs_count != -1) { + if (callxargs_count <= 15) { + o << AsmOp::Custom(PSTRING() << callxargs_count << " -1 CALLXARGS"); + } else { + func_assert(callxargs_count <= 254); + o << AsmOp::Custom(PSTRING() << callxargs_count << " PUSHINT -1 PUSHINT CALLXVARARGS"); + } + } else { o << "EXECUTE"; } } diff --git a/crypto/func/gen-abscode.cpp b/crypto/func/gen-abscode.cpp index 6af8e5b87..9989d10c4 100644 --- a/crypto/func/gen-abscode.cpp +++ b/crypto/func/gen-abscode.cpp @@ -92,7 +92,7 @@ bool Expr::deduce_type(const Lexem& lem) { return true; } case _VarApply: { - assert(args.size() == 2); + func_assert(args.size() == 2); TypeExpr* fun_type = TypeExpr::new_map(args[1]->e_type, TypeExpr::new_hole()); try { unify(fun_type, args[0]->e_type); @@ -107,7 +107,7 @@ bool Expr::deduce_type(const Lexem& lem) { return true; } case _Letop: { - assert(args.size() == 2); + func_assert(args.size() == 2); try { // std::cerr << "in assignment: " << args[0]->e_type << " from " << args[1]->e_type << std::endl; unify(args[0]->e_type, args[1]->e_type); @@ -122,7 +122,7 @@ bool Expr::deduce_type(const Lexem& lem) { return true; } case _LetFirst: { - assert(args.size() == 2); + func_assert(args.size() == 2); TypeExpr* rhs_type = TypeExpr::new_tensor({args[0]->e_type, TypeExpr::new_hole()}); try { // std::cerr << "in implicit assignment of a modifying method: " << rhs_type << " and " << args[1]->e_type << std::endl; @@ -140,7 +140,7 @@ bool Expr::deduce_type(const Lexem& lem) { return true; } case _CondExpr: { - assert(args.size() == 3); + func_assert(args.size() == 3); auto flag_type = TypeExpr::new_atomic(_Int); try { unify(args[0]->e_type, flag_type); @@ -204,7 +204,7 @@ int Expr::predefine_vars() { } case _Var: if (!sym) { - assert(val < 0 && here.defined()); + func_assert(val < 0 && here.defined()); if (prohibited_var_names.count(sym::symbols.get_name(~val))) { throw src::ParseError{ here, PSTRING() << "symbol `" << sym::symbols.get_name(~val) << "` cannot be redefined as a variable"}; @@ -274,7 +274,7 @@ std::vector pre_compile_tensor(const std::vector args, CodeBl arg_order.resize(args.size()); std::iota(arg_order.begin(), arg_order.end(), 0); } - assert(args.size() == arg_order.size()); + func_assert(args.size() == arg_order.size()); std::vector> res_lists(args.size()); struct ModifiedVar { @@ -310,7 +310,7 @@ std::vector pre_compile_tensor(const std::vector args, CodeBl } for (const auto& list : res_lists) { for (var_idx_t v : list) { - assert(!code.vars.at(v).on_modification.empty()); + func_assert(!code.vars.at(v).on_modification.empty()); code.vars.at(v).on_modification.pop_back(); } } @@ -339,7 +339,7 @@ std::vector Expr::pre_compile(CodeBlob& code, std::vector(sym->value); std::vector res; if (func && func->arg_order.size() == args.size() && !(code.flags & CodeBlob::_ComputeAsmLtr)) { @@ -426,7 +426,7 @@ std::vector Expr::pre_compile(CodeBlob& code, std::vectorpre_compile(code); - assert(cond.size() == 1); + func_assert(cond.size() == 1); auto rvect = new_tmp_vect(code); Op& if_op = code.emplace_back(here, Op::_If, cond); code.push_set_cur(if_op.block0); diff --git a/crypto/func/optimize.cpp b/crypto/func/optimize.cpp index f8dc3d827..74bb97ecd 100644 --- a/crypto/func/optimize.cpp +++ b/crypto/func/optimize.cpp @@ -61,9 +61,9 @@ void Optimizer::apply() { if (!p_ && !q_) { return; } - assert(p_ > 0 && p_ <= l_ && q_ >= 0 && q_ <= n && l_ <= n); + func_assert(p_ > 0 && p_ <= l_ && q_ >= 0 && q_ <= n && l_ <= n); for (int i = p_; i < l_; i++) { - assert(op_[i]); + func_assert(op_[i]); op_cons_[i]->car = std::move(op_[i]); op_cons_[i] = nullptr; } @@ -71,7 +71,7 @@ void Optimizer::apply() { code_ = std::move(code_->cdr); } for (int j = q_ - 1; j >= 0; j--) { - assert(oq_[j]); + func_assert(oq_[j]); oq_[j]->indent = indent_; code_ = AsmOpCons::cons(std::move(oq_[j]), std::move(code_)); } @@ -246,7 +246,7 @@ bool Optimizer::rewrite_const_push_xchgs() { } show_left(); auto c_op = std::move(op_[0]); - assert(c_op->is_gconst()); + func_assert(c_op->is_gconst()); StackTransform t; q_ = 0; int pos = 0; @@ -265,31 +265,31 @@ bool Optimizer::rewrite_const_push_xchgs() { if (b > pos) { oq_[q_]->b = b - 1; } - assert(apply_op(t, *oq_[q_])); + func_assert(apply_op(t, *oq_[q_])); ++q_; } } else { - assert(op_[i]->is_push(&a)); - assert(a != pos); + func_assert(op_[i]->is_push(&a)); + func_assert(a != pos); oq_[q_] = std::move(op_[i]); if (a > pos) { oq_[q_]->a = a - 1; } - assert(apply_op(t, *oq_[q_])); + func_assert(apply_op(t, *oq_[q_])); ++q_; ++pos; } } - assert(!pos); + func_assert(!pos); t.apply_push_newconst(); - assert(t <= tr_[p_ - 1]); + func_assert(t <= tr_[p_ - 1]); oq_[q_++] = std::move(c_op); show_right(); return true; } bool Optimizer::rewrite(int p, AsmOp&& new_op) { - assert(p > 0 && p <= l_); + func_assert(p > 0 && p <= l_); p_ = p; q_ = 1; show_left(); @@ -300,7 +300,7 @@ bool Optimizer::rewrite(int p, AsmOp&& new_op) { } bool Optimizer::rewrite(int p, AsmOp&& new_op1, AsmOp&& new_op2) { - assert(p > 1 && p <= l_); + func_assert(p > 1 && p <= l_); p_ = p; q_ = 2; show_left(); @@ -313,7 +313,7 @@ bool Optimizer::rewrite(int p, AsmOp&& new_op1, AsmOp&& new_op2) { } bool Optimizer::rewrite(int p, AsmOp&& new_op1, AsmOp&& new_op2, AsmOp&& new_op3) { - assert(p > 2 && p <= l_); + func_assert(p > 2 && p <= l_); p_ = p; q_ = 3; show_left(); @@ -328,7 +328,7 @@ bool Optimizer::rewrite(int p, AsmOp&& new_op1, AsmOp&& new_op2, AsmOp&& new_op3 } bool Optimizer::rewrite_nop() { - assert(p_ > 0 && p_ <= l_); + func_assert(p_ > 0 && p_ <= l_); q_ = 0; show_left(); show_right(); diff --git a/crypto/func/parse-func.cpp b/crypto/func/parse-func.cpp index 06c695bfc..15794c425 100644 --- a/crypto/func/parse-func.cpp +++ b/crypto/func/parse-func.cpp @@ -399,7 +399,7 @@ bool check_global_func(const Lexem& cur, sym_idx_t func_name = 0) { cur.loc.show_error(std::string{"undefined function `"} + symbols.get_name(func_name) + "`, defining a global function of unknown type"); def = sym::define_global_symbol(func_name, 0, cur.loc); - assert(def && "cannot define global function"); + func_assert(def && "cannot define global function"); ++undef_func_cnt; make_new_glob_func(def, TypeExpr::new_func()); // was: ... ::new_func() return true; @@ -1111,6 +1111,7 @@ blk_fl::val parse_do_stmt(Lexer& lex, CodeBlob& code) { } blk_fl::val parse_try_catch_stmt(Lexer& lex, CodeBlob& code) { + code.require_callxargs = true; lex.expect(_Try); Op& try_catch_op = code.emplace_back(lex.cur().loc, Op::_TryCatch); code.push_set_cur(try_catch_op.block0); @@ -1132,7 +1133,7 @@ blk_fl::val parse_try_catch_stmt(Lexer& lex, CodeBlob& code) { expr->predefine_vars(); expr->define_new_vars(code); try_catch_op.left = expr->pre_compile(code); - assert(try_catch_op.left.size() == 2); + func_assert(try_catch_op.left.size() == 2 || try_catch_op.left.size() == 1); blk_fl::val res1 = parse_block_stmt(lex, code); sym::close_scope(lex); code.close_pop_cur(lex.cur().loc); @@ -1295,7 +1296,7 @@ SymValAsmFunc* parse_asm_func_body(Lexer& lex, TypeExpr* func_type, const Formal } lex.next(); } - assert(arg_order.size() == (unsigned)tot_width); + func_assert(arg_order.size() == (unsigned)tot_width); } if (lex.tp() == _Mapsto) { lex.expect(_Mapsto); @@ -1487,7 +1488,7 @@ void parse_func_def(Lexer& lex) { std::cerr << "function " << func_name.str << " : " << func_type << std::endl; } SymDef* func_sym = sym::define_global_symbol(func_name.val, 0, loc); - assert(func_sym); + func_assert(func_sym); SymValFunc* func_sym_val = dynamic_cast(func_sym->value); if (func_sym->value) { if (func_sym->value->type != SymVal::_Func || !func_sym_val) { diff --git a/crypto/func/stack-transform.cpp b/crypto/func/stack-transform.cpp index 4d9b6a5fc..b52906086 100644 --- a/crypto/func/stack-transform.cpp +++ b/crypto/func/stack-transform.cpp @@ -183,7 +183,7 @@ bool StackTransform::is_permutation() const { if (!is_valid() || d) { return false; } - assert(n <= max_n); + func_assert(n <= max_n); std::array X, Y; for (int i = 0; i < n; i++) { X[i] = A[i].first; diff --git a/crypto/func/unify-types.cpp b/crypto/func/unify-types.cpp index dfa1f6029..f9b639c0e 100644 --- a/crypto/func/unify-types.cpp +++ b/crypto/func/unify-types.cpp @@ -132,7 +132,7 @@ void TypeExpr::replace_with(TypeExpr* te2) { } bool TypeExpr::remove_indirect(TypeExpr*& te, TypeExpr* forbidden) { - assert(te); + func_assert(te); while (te->constr == te_Indirect) { te = te->args[0]; } @@ -147,8 +147,8 @@ bool TypeExpr::remove_indirect(TypeExpr*& te, TypeExpr* forbidden) { } std::vector TypeExpr::remove_forall(TypeExpr*& te) { - assert(te && te->constr == te_ForAll); - assert(te->args.size() >= 1); + func_assert(te && te->constr == te_ForAll); + func_assert(te->args.size() >= 1); std::vector new_vars; for (std::size_t i = 1; i < te->args.size(); i++) { new_vars.push_back(new_hole(1)); @@ -162,8 +162,8 @@ std::vector TypeExpr::remove_forall(TypeExpr*& te) { } bool TypeExpr::remove_forall_in(TypeExpr*& te, TypeExpr* te2, const std::vector& new_vars) { - assert(te); - assert(te2 && te2->constr == te_ForAll); + func_assert(te); + func_assert(te2 && te2->constr == te_ForAll); if (te->constr == te_Var) { for (std::size_t i = 0; i < new_vars.size(); i++) { if (te == te2->args[i + 1]) { @@ -277,7 +277,7 @@ std::ostream& TypeExpr::print(std::ostream& os, int lex_level) { return os << "]"; } case te_Map: { - assert(args.size() == 2); + func_assert(args.size() == 2); if (lex_level > 0) { os << "("; } @@ -290,7 +290,7 @@ std::ostream& TypeExpr::print(std::ostream& os, int lex_level) { return os; } case te_ForAll: { - assert(args.size() >= 1); + func_assert(args.size() >= 1); if (lex_level > 0) { os << '('; } @@ -343,11 +343,11 @@ void check_update_widths(TypeExpr* te1, TypeExpr* te2) { check_width_compat(te1, te2); te1->minw = te2->minw = std::max(te1->minw, te2->minw); te1->maxw = te2->maxw = std::min(te1->maxw, te2->maxw); - assert(te1->minw <= te1->maxw); + func_assert(te1->minw <= te1->maxw); } void unify(TypeExpr*& te1, TypeExpr*& te2) { - assert(te1 && te2); + func_assert(te1 && te2); // std::cerr << "unify( " << te1 << " , " << te2 << " )\n"; while (te1->constr == TypeExpr::te_Indirect) { te1 = te1->args[0]; @@ -390,7 +390,7 @@ void unify(TypeExpr*& te1, TypeExpr*& te2) { } if (te1->constr == TypeExpr::te_Unknown) { if (te2->constr == TypeExpr::te_Unknown) { - assert(te1->value != te2->value); + func_assert(te1->value != te2->value); } if (!TypeExpr::remove_indirect(te2, te1)) { throw UnifyError{te1, te2, "type unification results in an infinite cyclic type"}; From dad980ed09b7aec0c395751cb5671c716b3f5ae2 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Mon, 15 May 2023 16:35:23 +0300 Subject: [PATCH 19/20] Fix STSLICECONST, bump Asm.fif version to 0.4.4 (#700) --- crypto/fift/lib/Asm.fif | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crypto/fift/lib/Asm.fif b/crypto/fift/lib/Asm.fif index 937358e08..64b850096 100644 --- a/crypto/fift/lib/Asm.fif +++ b/crypto/fift/lib/Asm.fif @@ -2,7 +2,7 @@ library TVM_Asm // simple TVM Assembler namespace Asm Asm definitions -"0.4.3" constant asm-fif-version +"0.4.4" constant asm-fif-version variable @atend variable @was-split @@ -578,7 +578,7 @@ x{CF3F} @Defop BCHKBITREFSQ x{CF40} @Defop STZEROES x{CF41} @Defop STONES x{CF42} @Defop STSAME -{ tuck sbitrefs swap 15 + swap @havebitrefs not +{ tuck sbitrefs swap 22 + swap @havebitrefs not { swap PUSHSLICE STSLICER } { over sbitrefs 2dup 57 3 2x<= { rot x{CFC_} s, swap 2 u, over 6 + 3 >> tuck 3 u, 3 roll s, From d5cd5485025fd8389aea99c1831a4fc587071509 Mon Sep 17 00:00:00 2001 From: EmelyanenkoK Date: Tue, 16 May 2023 13:28:23 +0300 Subject: [PATCH 20/20] Add 2023.05 changelog --- Changelog.md | 9 +++++++++ recent_changelog.md | 14 ++++++++------ 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/Changelog.md b/Changelog.md index 9fd649c40..0f0cd833a 100644 --- a/Changelog.md +++ b/Changelog.md @@ -1,3 +1,12 @@ +## 2023.05 Update +1. Archive manager optimization +2. A series of catchain (basic consensus protocol) security improvements +3. Update for Fift libraries and FunC: better error-handling, fixes for `catch` stack recovery +4. A series of out message queue handling optimization (already deployed during emergency upgrades between releases) +5. Improvement of binaries portability + +Besides the work of the core team, this update is based on the efforts of @aleksej-paschenko (portability improvement), [Disintar team](https://github.com/disintar/) (archive manager optimization) and [sec3-service](https://github.com/sec3-service) security auditors (funC improvements). + ## 2023.04 Update 1. CPU load optimization: previous DHT reconnect policy was too aggressive 2. Network throughput improvements: granular control on external message broadcast, optimize celldb GC, adjust state serialization and block downloading timings, rldp2 for states and archives diff --git a/recent_changelog.md b/recent_changelog.md index 592976697..39ca5c942 100644 --- a/recent_changelog.md +++ b/recent_changelog.md @@ -1,6 +1,8 @@ -## 03.2023 Update -1. Improvement of ADNL connection stability -2. Transaction emulator support and getAccountStateByTransaction method -3. Fixes of typos, undefined behavior and timer warnings -4. Handling incorrect integer literal values in funC; funC version bumped to 0.4.2 -5. FunC Mathlib \ No newline at end of file +## 2023.05 Update +1. Archive manager optimization +2. A series of catchain (basic consensus protocol) security improvements +3. Update for Fift libraries and FunC: better error-handling, fixes for `catch` stack recovery +4. A series of out message queue handling optimization (already deployed during emergency upgrades between releases) +5. Improvement of binaries portability + +Besides the work of the core team, this update is based on the efforts of @aleksej-paschenko (portability improvement), [Disintar team](https://github.com/disintar/) (archive manager optimization) and [sec3-service](https://github.com/sec3-service) security auditors (funC improvements).