diff --git a/.github/workflows/create-release.yml b/.github/workflows/create-release.yml index 85d0dc259..cb6651220 100644 --- a/.github/workflows/create-release.yml +++ b/.github/workflows/create-release.yml @@ -27,21 +27,21 @@ jobs: workflow_conclusion: success skip_unpack: false -# - name: Download Linux arm64 artifacts -# uses: dawidd6/action-download-artifact@v2 -# with: -# workflow: ton-aarch64-linux.yml -# path: artifacts -# workflow_conclusion: success -# skip_unpack: true -# -# - name: Download and unzip Linux arm64 artifacts -# uses: dawidd6/action-download-artifact@v2 -# with: -# workflow: ton-aarch64-linux.yml -# path: artifacts -# workflow_conclusion: success -# skip_unpack: false + # - name: Download Linux arm64 artifacts + # uses: dawidd6/action-download-artifact@v2 + # with: + # workflow: ton-aarch64-linux.yml + # path: artifacts + # workflow_conclusion: success + # skip_unpack: true + # + # - name: Download and unzip Linux arm64 artifacts + # uses: dawidd6/action-download-artifact@v2 + # with: + # workflow: ton-aarch64-linux.yml + # path: artifacts + # workflow_conclusion: success + # skip_unpack: false - name: Download Mac x86-64 artifacts uses: dawidd6/action-download-artifact@v2 @@ -59,21 +59,21 @@ jobs: workflow_conclusion: success skip_unpack: false -# - name: Download Mac arm64 artifacts -# uses: dawidd6/action-download-artifact@v2 -# with: -# workflow: ton-aarch64-macos.yml -# path: artifacts -# workflow_conclusion: success -# skip_unpack: true -# -# - name: Download and unzip Mac arm64 artifacts -# uses: dawidd6/action-download-artifact@v2 -# with: -# workflow: ton-aarch64-macos.yml -# path: artifacts -# workflow_conclusion: success -# skip_unpack: false + # - name: Download Mac arm64 artifacts + # uses: dawidd6/action-download-artifact@v2 + # with: + # workflow: ton-aarch64-macos.yml + # path: artifacts + # workflow_conclusion: success + # skip_unpack: true + # + # - name: Download and unzip Mac arm64 artifacts + # uses: dawidd6/action-download-artifact@v2 + # with: + # workflow: ton-aarch64-macos.yml + # path: artifacts + # workflow_conclusion: success + # skip_unpack: false - name: Download Windows artifacts uses: dawidd6/action-download-artifact@v2 @@ -104,26 +104,35 @@ jobs: tree artifacts -# create release + # create release - name: Read Changelog.md and use it as a body of new release id: read_release shell: bash run: | - r=$(cat Changelog.md) + r=$(cat recent_changelog.md) r="${r//'%'/'%25'}" r="${r//$'\n'/'%0A'}" r="${r//$'\r'/'%0D'}" echo "::set-output name=CHANGELOG_BODY::$r" - - name: Get current date - id: date - run: echo "::set-output name=date::$(date +'%Y.%m')" + - name: Get next tag + id: tag + run: | + git fetch --all --tags + git tag -l + NEW_TAG=v$(date +'%Y.%m') + FOUND=$(git tag -l | grep $NEW_TAG | wc -l) + if [ $FOUND -eq 0 ]; then + echo "TAG=$NEW_TAG" >> $GITHUB_OUTPUT + else + echo "TAG=$NEW_TAG-$FOUND" >> $GITHUB_OUTPUT + fi - name: Get registration token id: getRegToken run: | - curl -X POST -H \"Accept: application/vnd.github+json\" -H 'Authorization: token ${{ secrets.GITHUB_TOKEN }}' https://api.github.com/repos/ton-blockchain/ton/actions/runners/registration-token + curl -X POST -H \"Accept: application/vnd.github+json\" -H 'Authorization: token ${{ secrets.GITHUB_TOKEN }}' https://api.github.com/repos/ton-blockchain/ton/actions/runners/registration-token - name: Create release id: create_release @@ -131,16 +140,16 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: - tag_name: v${{ steps.date.outputs.date }} - release_name: v${{ steps.date.outputs.date }} + tag_name: ${{ steps.tag.outputs.TAG }} + release_name: TON ${{ steps.tag.outputs.TAG }} body: | ${{ steps.read_release.outputs.CHANGELOG_BODY }} draft: false prerelease: false -# upload + # upload -# win + # win - name: Upload Windows 2019 artifacts uses: svenstaro/upload-release-action@v2 @@ -148,7 +157,7 @@ jobs: repo_token: ${{ secrets.GITHUB_TOKEN }} file: artifacts/ton-win-binaries.zip asset_name: ton-win-x86-64.zip - tag: v${{ steps.date.outputs.date }} + tag: ${{ steps.tag.outputs.TAG }} - name: Upload Windows 2019 single artifact - fift uses: svenstaro/upload-release-action@v2 @@ -156,7 +165,7 @@ jobs: repo_token: ${{ secrets.GITHUB_TOKEN }} file: artifacts/ton-win-binaries/fift.exe asset_name: fift.exe - tag: v${{ steps.date.outputs.date }} + tag: ${{ steps.tag.outputs.TAG }} - name: Upload Windows 2019 single artifact - func uses: svenstaro/upload-release-action@v2 @@ -164,7 +173,7 @@ jobs: repo_token: ${{ secrets.GITHUB_TOKEN }} file: artifacts/ton-win-binaries/func.exe asset_name: func.exe - tag: v${{ steps.date.outputs.date }} + tag: ${{ steps.tag.outputs.TAG }} - name: Upload Windows 2019 single artifact - lite-client uses: svenstaro/upload-release-action@v2 @@ -172,7 +181,7 @@ jobs: repo_token: ${{ secrets.GITHUB_TOKEN }} file: artifacts/ton-win-binaries/lite-client.exe asset_name: lite-client.exe - tag: v${{ steps.date.outputs.date }} + tag: ${{ steps.tag.outputs.TAG }} - name: Upload Windows 2019 single artifact - rldp-http-proxy uses: svenstaro/upload-release-action@v2 @@ -180,7 +189,7 @@ jobs: repo_token: ${{ secrets.GITHUB_TOKEN }} file: artifacts/ton-win-binaries/rldp-http-proxy.exe asset_name: rldp-http-proxy.exe - tag: v${{ steps.date.outputs.date }} + tag: ${{ steps.tag.outputs.TAG }} - name: Upload Windows 2019 single artifact - http-proxy uses: svenstaro/upload-release-action@v2 @@ -188,7 +197,7 @@ jobs: repo_token: ${{ secrets.GITHUB_TOKEN }} file: artifacts/ton-win-binaries/http-proxy.exe asset_name: http-proxy.exe - tag: v${{ steps.date.outputs.date }} + tag: ${{ steps.tag.outputs.TAG }} - name: Upload Windows 2019 single artifact - storage-daemon-cli uses: svenstaro/upload-release-action@v2 @@ -196,7 +205,7 @@ jobs: repo_token: ${{ secrets.GITHUB_TOKEN }} file: artifacts/ton-win-binaries/storage-daemon-cli.exe asset_name: storage-daemon-cli.exe - tag: v${{ steps.date.outputs.date }} + tag: ${{ steps.tag.outputs.TAG }} - name: Upload Windows 2019 single artifact - tonlibjson uses: svenstaro/upload-release-action@v2 @@ -204,7 +213,7 @@ jobs: repo_token: ${{ secrets.GITHUB_TOKEN }} file: artifacts/ton-win-binaries/tonlibjson.dll asset_name: tonlibjson.dll - tag: v${{ steps.date.outputs.date }} + tag: ${{ steps.tag.outputs.TAG }} - name: Upload Windows 2019 single artifact - tonlib-cli uses: svenstaro/upload-release-action@v2 @@ -212,9 +221,9 @@ jobs: repo_token: ${{ secrets.GITHUB_TOKEN }} file: artifacts/ton-win-binaries/tonlib-cli.exe asset_name: tonlib-cli.exe - tag: v${{ steps.date.outputs.date }} + tag: ${{ steps.tag.outputs.TAG }} -# mac x86-64 + # mac x86-64 - name: Upload Mac x86-64 artifacts uses: svenstaro/upload-release-action@v2 @@ -222,7 +231,7 @@ jobs: repo_token: ${{ secrets.GITHUB_TOKEN }} file: artifacts/ton-x86_64-macos-binaries.zip asset_name: ton-mac-x86-64.zip - tag: v${{ steps.date.outputs.date }} + tag: ${{ steps.tag.outputs.TAG }} - name: Upload Mac x86-64 single artifact - fift uses: svenstaro/upload-release-action@v2 @@ -230,7 +239,7 @@ jobs: repo_token: ${{ secrets.GITHUB_TOKEN }} file: artifacts/ton-x86_64-macos-binaries/fift asset_name: fift-mac-x86-64 - tag: v${{ steps.date.outputs.date }} + tag: ${{ steps.tag.outputs.TAG }} - name: Upload Mac x86-64 single artifact - func uses: svenstaro/upload-release-action@v2 @@ -238,7 +247,7 @@ jobs: repo_token: ${{ secrets.GITHUB_TOKEN }} file: artifacts/ton-x86_64-macos-binaries/func asset_name: func-mac-x86-64 - tag: v${{ steps.date.outputs.date }} + tag: ${{ steps.tag.outputs.TAG }} - name: Upload Mac x86-64 single artifact - lite-client uses: svenstaro/upload-release-action@v2 @@ -246,7 +255,7 @@ jobs: repo_token: ${{ secrets.GITHUB_TOKEN }} file: artifacts/ton-x86_64-macos-binaries/lite-client asset_name: lite-client-mac-x86-64 - tag: v${{ steps.date.outputs.date }} + tag: ${{ steps.tag.outputs.TAG }} - name: Upload Mac x86-64 single artifact - rldp-http-proxy uses: svenstaro/upload-release-action@v2 @@ -254,7 +263,7 @@ jobs: repo_token: ${{ secrets.GITHUB_TOKEN }} file: artifacts/ton-x86_64-macos-binaries/rldp-http-proxy asset_name: rldp-http-proxy-mac-x86-64 - tag: v${{ steps.date.outputs.date }} + tag: ${{ steps.tag.outputs.TAG }} - name: Upload Mac x86-64 single artifact - http-proxy uses: svenstaro/upload-release-action@v2 @@ -262,7 +271,7 @@ jobs: repo_token: ${{ secrets.GITHUB_TOKEN }} file: artifacts/ton-x86_64-macos-binaries/http-proxy asset_name: http-proxy-mac-x86-64 - tag: v${{ steps.date.outputs.date }} + tag: ${{ steps.tag.outputs.TAG }} - name: Upload Mac x86-64 single artifact - storage-daemon-cli uses: svenstaro/upload-release-action@v2 @@ -270,7 +279,7 @@ jobs: repo_token: ${{ secrets.GITHUB_TOKEN }} file: artifacts/ton-x86_64-macos-binaries/storage-daemon-cli asset_name: storage-daemon-cli-mac-x86-64 - tag: v${{ steps.date.outputs.date }} + tag: ${{ steps.tag.outputs.TAG }} - name: Upload Mac x86-64 single artifact - tonlibjson uses: svenstaro/upload-release-action@v2 @@ -278,7 +287,7 @@ jobs: repo_token: ${{ secrets.GITHUB_TOKEN }} file: artifacts/ton-x86_64-macos-binaries/libtonlibjson.dylib asset_name: tonlibjson-mac-x86-64.dylib - tag: v${{ steps.date.outputs.date }} + tag: ${{ steps.tag.outputs.TAG }} - name: Upload Mac x86-64 single artifact - tonlib-cli uses: svenstaro/upload-release-action@v2 @@ -286,9 +295,9 @@ jobs: repo_token: ${{ secrets.GITHUB_TOKEN }} file: artifacts/ton-x86_64-macos-binaries/tonlib-cli asset_name: tonlib-cli-mac-x86-64 - tag: v${{ steps.date.outputs.date }} + tag: ${{ steps.tag.outputs.TAG }} -# linux x86-64 + # linux x86-64 - name: Upload Linux x86-64 artifacts uses: svenstaro/upload-release-action@v2 @@ -296,7 +305,7 @@ jobs: repo_token: ${{ secrets.GITHUB_TOKEN }} file: artifacts/ton-x86_64-linux-binaries.zip asset_name: ton-linux-x86_64.zip - tag: v${{ steps.date.outputs.date }} + tag: ${{ steps.tag.outputs.TAG }} - name: Upload Linux x86-64 single artifact - fift uses: svenstaro/upload-release-action@v2 @@ -304,7 +313,7 @@ jobs: repo_token: ${{ secrets.GITHUB_TOKEN }} file: artifacts/ton-x86_64-linux-binaries/fift asset_name: fift-linux-x86_64 - tag: v${{ steps.date.outputs.date }} + tag: ${{ steps.tag.outputs.TAG }} - name: Upload Linux x86-64 single artifact - func uses: svenstaro/upload-release-action@v2 @@ -312,7 +321,7 @@ jobs: repo_token: ${{ secrets.GITHUB_TOKEN }} file: artifacts/ton-x86_64-linux-binaries/func asset_name: func-linux-x86_64 - tag: v${{ steps.date.outputs.date }} + tag: ${{ steps.tag.outputs.TAG }} - name: Upload Linux x86-64 single artifact - lite-client uses: svenstaro/upload-release-action@v2 @@ -320,7 +329,7 @@ jobs: repo_token: ${{ secrets.GITHUB_TOKEN }} file: artifacts/ton-x86_64-linux-binaries/lite-client asset_name: lite-client-linux-x86_64 - tag: v${{ steps.date.outputs.date }} + tag: ${{ steps.tag.outputs.TAG }} - name: Upload Linux x86-64 single artifact - rldp-http-proxy uses: svenstaro/upload-release-action@v2 @@ -328,7 +337,7 @@ jobs: repo_token: ${{ secrets.GITHUB_TOKEN }} file: artifacts/ton-x86_64-linux-binaries/rldp-http-proxy asset_name: rldp-http-proxy-linux-x86_64 - tag: v${{ steps.date.outputs.date }} + tag: ${{ steps.tag.outputs.TAG }} - name: Upload Linux x86-64 single artifact - http-proxy uses: svenstaro/upload-release-action@v2 @@ -336,7 +345,7 @@ jobs: repo_token: ${{ secrets.GITHUB_TOKEN }} file: artifacts/ton-x86_64-linux-binaries/http-proxy asset_name: http-proxy-linux-x86_64 - tag: v${{ steps.date.outputs.date }} + tag: ${{ steps.tag.outputs.TAG }} - name: Upload Linux x86-64 single artifact - storage-daemon-cli uses: svenstaro/upload-release-action@v2 @@ -344,7 +353,7 @@ jobs: repo_token: ${{ secrets.GITHUB_TOKEN }} file: artifacts/ton-x86_64-linux-binaries/storage-daemon-cli asset_name: storage-daemon-cli-linux-x86_64 - tag: v${{ steps.date.outputs.date }} + tag: ${{ steps.tag.outputs.TAG }} - name: Upload Linux x86-64 single artifact - tonlibjson uses: svenstaro/upload-release-action@v2 @@ -352,7 +361,7 @@ jobs: repo_token: ${{ secrets.GITHUB_TOKEN }} file: artifacts/ton-x86_64-linux-binaries/libtonlibjson.so.0.5 asset_name: tonlibjson-linux-x86_64.so - tag: v${{ steps.date.outputs.date }} + tag: ${{ steps.tag.outputs.TAG }} - name: Upload Linux x86-64 single artifact - tonlib-cli uses: svenstaro/upload-release-action@v2 @@ -360,23 +369,23 @@ jobs: repo_token: ${{ secrets.GITHUB_TOKEN }} file: artifacts/ton-x86_64-linux-binaries/tonlib-cli asset_name: tonlib-cli-linux-x86_64 - tag: v${{ steps.date.outputs.date }} - -# - name: Upload Linux arm64 artifacts -# uses: svenstaro/upload-release-action@v2 -# with: -# repo_token: ${{ secrets.GITHUB_TOKEN }} -# file: artifacts/ton-aarch64-linux-binaries.zip -# asset_name: ton-linux-arm64.zip -# tag: v${{ steps.date.outputs.date }} -# -# - name: Upload Mac arm64 artifacts -# uses: svenstaro/upload-release-action@v2 -# with: -# repo_token: ${{ secrets.GITHUB_TOKEN }} -# file: artifacts/ton-aarch64-macos-binaries -# asset_name: ton-mac-arm64.zip -# tag: v${{ steps.date.outputs.date }} + tag: ${{ steps.tag.outputs.TAG }} + + # - name: Upload Linux arm64 artifacts + # uses: svenstaro/upload-release-action@v2 + # with: + # repo_token: ${{ secrets.GITHUB_TOKEN }} + # file: artifacts/ton-aarch64-linux-binaries.zip + # asset_name: ton-linux-arm64.zip + # tag: ${{ steps.tag.outputs.TAG }} + # + # - name: Upload Mac arm64 artifacts + # uses: svenstaro/upload-release-action@v2 + # with: + # repo_token: ${{ secrets.GITHUB_TOKEN }} + # file: artifacts/ton-aarch64-macos-binaries + # asset_name: ton-mac-arm64.zip + # tag: ${{ steps.tag.outputs.TAG }} - name: Upload WASM artifacts uses: svenstaro/upload-release-action@v2 @@ -384,4 +393,4 @@ jobs: repo_token: ${{ secrets.GITHUB_TOKEN }} file: artifacts/ton-wasm-binaries.zip asset_name: ton-wasm-binaries.zip - tag: v${{ steps.date.outputs.date }} + tag: ${{ steps.tag.outputs.TAG }} diff --git a/.github/workflows/ubuntu-22.04-compile.yml b/.github/workflows/ubuntu-22.04-compile.yml index d27d1b4da..8032e498a 100644 --- a/.github/workflows/ubuntu-22.04-compile.yml +++ b/.github/workflows/ubuntu-22.04-compile.yml @@ -27,9 +27,19 @@ jobs: export CC=$(which clang) export CXX=$(which clang++) export CCACHE_DISABLE=1 + + git clone https://github.com/openssl/openssl openssl_1_1_1 + cd openssl_1_1_1 + git checkout OpenSSL_1_1_1-stable + ./config + make build_libs -j4 + + cd .. + rootPath=`pwd` mkdir build cd build - cmake -GNinja -DCMAKE_BUILD_TYPE=Release -DPORTABLE=1 -DTON_ARCH= -DCMAKE_CXX_FLAGS="-mavx2" .. + + cmake -GNinja -DOPENSSL_FOUND=1 -DOPENSSL_INCLUDE_DIR=$rootPath/openssl_1_1_1/include -DOPENSSL_CRYPTO_LIBRARY=$rootPath/openssl_1_1_1/libcrypto.a -DCMAKE_BUILD_TYPE=Release -DPORTABLE=1 -DTON_ARCH= -DCMAKE_CXX_FLAGS="-mavx2" .. ninja storage-daemon storage-daemon-cli fift func tonlib tonlibjson tonlib-cli validator-engine lite-client pow-miner validator-engine-console generate-random-id json2tlo dht-server http-proxy rldp-http-proxy adnl-proxy create-state - name: Find & copy binaries diff --git a/.github/workflows/ubuntu-compile.yml b/.github/workflows/ubuntu-compile.yml index dd246a39e..b303e143b 100644 --- a/.github/workflows/ubuntu-compile.yml +++ b/.github/workflows/ubuntu-compile.yml @@ -30,9 +30,20 @@ jobs: export CC=$(which clang) export CXX=$(which clang++) export CCACHE_DISABLE=1 + mkdir build-${{ matrix.os }} cd build-${{ matrix.os }} - cmake -GNinja -DCMAKE_BUILD_TYPE=Release -DPORTABLE=1 -DTON_ARCH= -DCMAKE_CXX_FLAGS="-mavx2" .. + + git clone https://github.com/openssl/openssl openssl_1_1_1 + cd openssl_1_1_1 + git checkout OpenSSL_1_1_1-stable + ./config + make build_libs -j4 + + cd .. + buildPath=`pwd` + + cmake -GNinja -DOPENSSL_FOUND=1 -DOPENSSL_INCLUDE_DIR=$buildPath/openssl_1_1_1/include -DOPENSSL_CRYPTO_LIBRARY=$buildPath/openssl_1_1_1/libcrypto.a -DCMAKE_BUILD_TYPE=Release -DPORTABLE=1 -DTON_ARCH= -DCMAKE_CXX_FLAGS="-mavx2" .. ninja storage-daemon storage-daemon-cli fift func tonlib tonlibjson tonlib-cli validator-engine lite-client pow-miner validator-engine-console generate-random-id json2tlo dht-server http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork - name: Find & copy binaries diff --git a/.github/workflows/win-2019-compile.yml b/.github/workflows/win-2019-compile.yml index bd174a15c..b0c999714 100644 --- a/.github/workflows/win-2019-compile.yml +++ b/.github/workflows/win-2019-compile.yml @@ -48,18 +48,13 @@ jobs: curl -Lo libmicrohttpd-latest-w32-bin.zip https://ftpmirror.gnu.org/libmicrohttpd/libmicrohttpd-latest-w32-bin.zip unzip libmicrohttpd-latest-w32-bin.zip - - name: Install pre-compiled Readline Win64 - run: | - curl -Lo readline-5.0-1-lib.zip https://github.com/neodiX42/precompiled-openssl-win64/raw/main/readline-5.0-1-lib.zip - unzip readline-5.0-1-lib.zip - - name: Compile run: | set root=%cd% echo %root% mkdir build cd build - cmake -DREADLINE_INCLUDE_DIR=%root%\readline-5.0-1-lib\include\readline -DREADLINE_LIBRARY=%root%\readline-5.0-1-lib\lib\readline.lib -DZLIB_FOUND=1 -DMHD_FOUND=1 -DMHD_LIBRARY=%root%\libmicrohttpd-0.9.75-w32-bin\x86_64\VS2019\Release-static\libmicrohttpd.lib -DMHD_INCLUDE_DIR=%root%\libmicrohttpd-0.9.75-w32-bin\x86_64\VS2019\Release-static -DZLIB_INCLUDE_DIR=%root%\zlib -DZLIB_LIBRARY=%root%\zlib\contrib\vstudio\vc14\x64\ZlibStatReleaseWithoutAsm\zlibstat.lib -DOPENSSL_FOUND=1 -DOPENSSL_INCLUDE_DIR=%root%/openssl-1.1/x64/include -DOPENSSL_CRYPTO_LIBRARY=%root%/openssl-1.1/x64/lib/libcrypto.lib -DCMAKE_CXX_FLAGS="/DTD_WINDOWS=1 /EHsc /bigobj /W0" .. + cmake -DPORTABLE=1 -DZLIB_FOUND=1 -DMHD_FOUND=1 -DMHD_LIBRARY=%root%\libmicrohttpd-0.9.75-w32-bin\x86_64\VS2019\Release-static\libmicrohttpd.lib -DMHD_INCLUDE_DIR=%root%\libmicrohttpd-0.9.75-w32-bin\x86_64\VS2019\Release-static -DZLIB_INCLUDE_DIR=%root%\zlib -DZLIB_LIBRARY=%root%\zlib\contrib\vstudio\vc14\x64\ZlibStatReleaseWithoutAsm\zlibstat.lib -DOPENSSL_FOUND=1 -DOPENSSL_INCLUDE_DIR=%root%/openssl-1.1/x64/include -DOPENSSL_CRYPTO_LIBRARY=%root%/openssl-1.1/x64/lib/libcrypto.lib -DCMAKE_CXX_FLAGS="/DTD_WINDOWS=1 /EHsc /bigobj /W0" .. cmake --build . --target storage-daemon storage-daemon-cli blockchain-explorer fift func tonlib tonlibjson tonlib-cli validator-engine lite-client pow-miner validator-engine-console generate-random-id json2tlo dht-server http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork --config Release - name: Show executables diff --git a/Changelog.md b/Changelog.md index 9fd649c40..0f0cd833a 100644 --- a/Changelog.md +++ b/Changelog.md @@ -1,3 +1,12 @@ +## 2023.05 Update +1. Archive manager optimization +2. A series of catchain (basic consensus protocol) security improvements +3. Update for Fift libraries and FunC: better error-handling, fixes for `catch` stack recovery +4. A series of out message queue handling optimization (already deployed during emergency upgrades between releases) +5. Improvement of binaries portability + +Besides the work of the core team, this update is based on the efforts of @aleksej-paschenko (portability improvement), [Disintar team](https://github.com/disintar/) (archive manager optimization) and [sec3-service](https://github.com/sec3-service) security auditors (funC improvements). + ## 2023.04 Update 1. CPU load optimization: previous DHT reconnect policy was too aggressive 2. Network throughput improvements: granular control on external message broadcast, optimize celldb GC, adjust state serialization and block downloading timings, rldp2 for states and archives diff --git a/catchain/catchain-receiver-interface.h b/catchain/catchain-receiver-interface.h index c8f1ef66d..bc02832ae 100644 --- a/catchain/catchain-receiver-interface.h +++ b/catchain/catchain-receiver-interface.h @@ -52,6 +52,7 @@ class CatChainReceiverInterface : public td::actor::Actor { td::BufferSlice query, td::uint64 max_answer_size, td::actor::ActorId via) = 0; virtual void send_custom_message_data(const PublicKeyHash &dst, td::BufferSlice query) = 0; + virtual void on_blame_processed(td::uint32 source_id) = 0; virtual void destroy() = 0; diff --git a/catchain/catchain-receiver-source.cpp b/catchain/catchain-receiver-source.cpp index 6a9e0777c..e758b3359 100644 --- a/catchain/catchain-receiver-source.cpp +++ b/catchain/catchain-receiver-source.cpp @@ -60,15 +60,15 @@ td::Result> CatChainReceiverSource::crea void CatChainReceiverSourceImpl::blame(td::uint32 fork, CatChainBlockHeight height) { blame(); - if (!blamed_heights_.empty()) { - if (blamed_heights_.size() <= fork) { - blamed_heights_.resize(fork + 1, 0); - } - if (blamed_heights_[fork] == 0 || blamed_heights_[fork] > height) { - VLOG(CATCHAIN_INFO) << this << ": blamed at " << fork << " " << height; - blamed_heights_[fork] = height; - } + // if (!blamed_heights_.empty()) { + if (blamed_heights_.size() <= fork) { + blamed_heights_.resize(fork + 1, 0); + } + if (blamed_heights_[fork] == 0 || blamed_heights_[fork] > height) { + VLOG(CATCHAIN_INFO) << this << ": blamed at " << fork << " " << height; + blamed_heights_[fork] = height; } + // } } void CatChainReceiverSourceImpl::blame() { @@ -144,7 +144,7 @@ void CatChainReceiverSourceImpl::on_new_block(CatChainReceivedBlock *block) { on_found_fork_proof(create_serialize_tl_object(block->export_tl_dep(), it->second->export_tl_dep()) .as_slice()); - chain_->add_prepared_event(fork_proof()); + chain_->on_found_fork_proof(id_, fork_proof()); } blame(); return; @@ -162,6 +162,15 @@ void CatChainReceiverSourceImpl::on_found_fork_proof(const td::Slice &proof) { } } +bool CatChainReceiverSourceImpl::allow_send_block(CatChainBlockHash hash) { + td::uint32 count = ++block_requests_count_[hash]; + if (count > MAX_BLOCK_REQUESTS) { + VLOG(CATCHAIN_INFO) << this << ": node requested block " << hash << " " << count << " times"; + return false; + } + return true; +} + } // namespace catchain } // namespace ton diff --git a/catchain/catchain-receiver-source.h b/catchain/catchain-receiver-source.h index e805c2a55..55035e779 100644 --- a/catchain/catchain-receiver-source.h +++ b/catchain/catchain-receiver-source.h @@ -61,6 +61,9 @@ class CatChainReceiverSource { virtual td::BufferSlice fork_proof() const = 0; virtual bool fork_is_found() const = 0; + // One block can be sent to one node only a limited number of times to prevent DoS + virtual bool allow_send_block(CatChainBlockHash hash) = 0; + static td::Result> create(CatChainReceiver *chain, PublicKey pub_key, adnl::AdnlNodeIdShort adnl_id, td::uint32 id); diff --git a/catchain/catchain-receiver-source.hpp b/catchain/catchain-receiver-source.hpp index 0785d8c2f..cf08c4219 100644 --- a/catchain/catchain-receiver-source.hpp +++ b/catchain/catchain-receiver-source.hpp @@ -111,6 +111,8 @@ class CatChainReceiverSourceImpl : public CatChainReceiverSource { return chain_; } + bool allow_send_block(CatChainBlockHash hash) override; + CatChainReceiverSourceImpl(CatChainReceiver *chain, PublicKey source, adnl::AdnlNodeIdShort adnl_id, td::uint32 id); private: @@ -130,6 +132,11 @@ class CatChainReceiverSourceImpl : public CatChainReceiverSource { CatChainBlockHeight delivered_height_ = 0; CatChainBlockHeight received_height_ = 0; + + std::map block_requests_count_; + // One block can be sent to one node up to 5 times + + static const td::uint32 MAX_BLOCK_REQUESTS = 5; }; } // namespace catchain diff --git a/catchain/catchain-receiver.cpp b/catchain/catchain-receiver.cpp index 488fbb9d5..482cfb43c 100644 --- a/catchain/catchain-receiver.cpp +++ b/catchain/catchain-receiver.cpp @@ -168,13 +168,6 @@ void CatChainReceiverImpl::receive_message_from_overlay(adnl::AdnlNodeIdShort sr return; } - /*auto S = get_source_by_hash(src); - CHECK(S != nullptr); - - if (S->blamed()) { - VLOG(CATCHAIN_INFO) << this << ": dropping block update from blamed source " << src; - return; - }*/ if (data.size() > opts_.max_serialized_block_size) { VLOG(CATCHAIN_WARNING) << this << ": dropping broken block from " << src << ": too big (size=" << data.size() << ", limit=" << opts_.max_serialized_block_size << ")"; @@ -197,19 +190,6 @@ void CatChainReceiverImpl::receive_broadcast_from_overlay(const PublicKeyHash &s callback_->on_broadcast(src, std::move(data)); } -/*void CatChainReceiverImpl::send_block(const PublicKeyHash &src, tl_object_ptr block, - td::BufferSlice payload) { - CHECK(read_db_); - CHECK(src == local_id_); - - validate_block_sync(block, payload.as_slice()).ensure(); - auto B = create_block(std::move(block), td::SharedSlice{payload.as_slice()}); - CHECK(B != nullptr); - - run_scheduler(); - CHECK(B->delivered()); -}*/ - CatChainReceivedBlock *CatChainReceiverImpl::create_block(tl_object_ptr block, td::SharedSlice payload) { if (block->height_ == 0) { @@ -267,21 +247,16 @@ td::Status CatChainReceiverImpl::validate_block_sync(const tl_object_ptr &block, const td::Slice &payload) const { - //LOG(INFO) << ton_api::to_string(block); TRY_STATUS_PREFIX(CatChainReceivedBlock::pre_validate_block(this, block, payload), "failed to validate block: "); + // After pre_validate_block, block->height_ > 0 + auto id = CatChainReceivedBlock::block_id(this, block, payload); + td::BufferSlice B = serialize_tl_object(id, true); - if (block->height_ > 0) { - auto id = CatChainReceivedBlock::block_id(this, block, payload); - td::BufferSlice B = serialize_tl_object(id, true); - - CatChainReceiverSource *S = get_source_by_hash(PublicKeyHash{id->src_}); - CHECK(S != nullptr); - Encryptor *E = S->get_encryptor_sync(); - CHECK(E != nullptr); - return E->check_signature(B.as_slice(), block->signature_.as_slice()); - } else { - return td::Status::OK(); - } + CatChainReceiverSource *S = get_source_by_hash(PublicKeyHash{id->src_}); + CHECK(S != nullptr); + Encryptor *E = S->get_encryptor_sync(); + CHECK(E != nullptr); + return E->check_signature(B.as_slice(), block->signature_.as_slice()); } void CatChainReceiverImpl::run_scheduler() { @@ -515,7 +490,6 @@ CatChainReceiverImpl::CatChainReceiverImpl(std::unique_ptr callback, } CHECK(local_idx_ != static_cast(ids.size())); - //std::sort(short_ids.begin(), short_ids.end()); auto F = create_tl_object(unique_hash, std::move(short_ids)); overlay_full_id_ = overlay::OverlayIdFull{serialize_tl_object(F, true)}; @@ -527,6 +501,8 @@ CatChainReceiverImpl::CatChainReceiverImpl(std::unique_ptr callback, blocks_[root_block_->get_hash()] = std::move(R); last_sent_block_ = root_block_; + blame_processed_.resize(sources_.size(), false); + choose_neighbours(); } @@ -704,7 +680,6 @@ void CatChainReceiverImpl::receive_query_from_overlay(adnl::AdnlNodeIdShort src, auto F = fetch_tl_object(data.clone(), true); if (F.is_error()) { callback_->on_custom_query(get_source_by_adnl_id(src)->get_hash(), std::move(data), std::move(promise)); - //LOG(WARNING) << this << ": unknown query from " << src; return; } auto f = F.move_as_ok(); @@ -717,68 +692,15 @@ void CatChainReceiverImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::cat if (it == blocks_.end() || it->second->get_height() == 0 || !it->second->initialized()) { promise.set_value(serialize_tl_object(create_tl_object(), true)); } else { - promise.set_value(serialize_tl_object(create_tl_object(it->second->export_tl()), - true, it->second->get_payload().as_slice())); - } -} - -void CatChainReceiverImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::catchain_getBlocks query, - td::Promise promise) { - if (query.blocks_.size() > MAX_QUERY_BLOCKS) { - promise.set_error(td::Status::Error(ErrorCode::protoviolation, "too many blocks")); - return; - } - td::int32 cnt = 0; - for (const CatChainBlockHash &b : query.blocks_) { - auto it = blocks_.find(b); - if (it != blocks_.end() && it->second->get_height() > 0) { - auto block = create_tl_object(it->second->export_tl()); - CHECK(!it->second->get_payload().empty()); - td::BufferSlice B = serialize_tl_object(block, true, it->second->get_payload().clone()); - CHECK(B.size() <= opts_.max_serialized_block_size); - td::actor::send_closure(overlay_manager_, &overlay::Overlays::send_message, src, - get_source(local_idx_)->get_adnl_id(), overlay_id_, std::move(B)); - cnt++; + CatChainReceiverSource *S = get_source_by_adnl_id(src); + CHECK(S != nullptr); + if (S->allow_send_block(it->second->get_hash())) { + promise.set_value(serialize_tl_object(create_tl_object(it->second->export_tl()), + true, it->second->get_payload().as_slice())); + } else { + promise.set_error(td::Status::Error("block was requested too many times")); } } - promise.set_value(serialize_tl_object(create_tl_object(cnt), true)); -} - -void CatChainReceiverImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::catchain_getBlockHistory query, - td::Promise promise) { - int64_t h = query.height_; - if (h <= 0) { - promise.set_error(td::Status::Error(ErrorCode::protoviolation, "not-positive height")); - return; - } - if (h > MAX_QUERY_HEIGHT) { - h = MAX_QUERY_HEIGHT; - } - std::set s{query.stop_if_.begin(), query.stop_if_.end()}; - - CatChainReceivedBlock *B = get_block(query.block_); - if (B == nullptr) { - promise.set_value(serialize_tl_object(create_tl_object(0), true)); - return; - } - if (static_cast(h) > B->get_height()) { - h = B->get_height(); - } - td::uint32 cnt = 0; - while (h-- > 0) { - if (s.find(B->get_hash()) != s.end()) { - break; - } - auto block = create_tl_object(B->export_tl()); - CHECK(!B->get_payload().empty()); - td::BufferSlice BB = serialize_tl_object(block, true, B->get_payload().as_slice()); - CHECK(BB.size() <= opts_.max_serialized_block_size); - td::actor::send_closure(overlay_manager_, &overlay::Overlays::send_message, src, - get_source(local_idx_)->get_adnl_id(), overlay_id_, std::move(BB)); - B = B->get_prev(); - cnt++; - } - promise.set_value(serialize_tl_object(create_tl_object(cnt), true)); } void CatChainReceiverImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::catchain_getDifference query, @@ -832,6 +754,8 @@ void CatChainReceiverImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::cat } } CHECK(right > 0); + CatChainReceiverSource *S0 = get_source_by_adnl_id(src); + CHECK(S0 != nullptr); for (td::uint32 i = 0; i < get_sources_cnt(); i++) { if (vt[i] >= 0 && my_vt[i] > vt[i]) { CatChainReceiverSource *S = get_source(i); @@ -839,12 +763,14 @@ void CatChainReceiverImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::cat while (t-- > 0) { CatChainReceivedBlock *M = S->get_block(++vt[i]); CHECK(M != nullptr); - auto block = create_tl_object(M->export_tl()); - CHECK(!M->get_payload().empty()); - td::BufferSlice BB = serialize_tl_object(block, true, M->get_payload().as_slice()); - CHECK(BB.size() <= opts_.max_serialized_block_size); - td::actor::send_closure(overlay_manager_, &overlay::Overlays::send_message, src, - get_source(local_idx_)->get_adnl_id(), overlay_id_, std::move(BB)); + if (S0->allow_send_block(M->get_hash())) { + auto block = create_tl_object(M->export_tl()); + CHECK(!M->get_payload().empty()); + td::BufferSlice BB = serialize_tl_object(block, true, M->get_payload().as_slice()); + CHECK(BB.size() <= opts_.max_serialized_block_size); + td::actor::send_closure(overlay_manager_, &overlay::Overlays::send_message, src, + get_source(local_idx_)->get_adnl_id(), overlay_id_, std::move(BB)); + } } } } @@ -1031,10 +957,15 @@ void CatChainReceiverImpl::written_unsafe_root_block(CatChainReceivedBlock *bloc void CatChainReceiverImpl::alarm() { alarm_timestamp() = td::Timestamp::never(); - if (next_sync_ && next_sync_.is_in_past()) { + if (next_sync_ && next_sync_.is_in_past() && get_sources_cnt() > 1) { next_sync_ = td::Timestamp::in(td::Random::fast(SYNC_INTERVAL_MIN, SYNC_INTERVAL_MAX)); for (unsigned i = 0; i < SYNC_ITERATIONS; i++) { - CatChainReceiverSource *S = get_source(td::Random::fast(0, static_cast(get_sources_cnt()) - 1)); + auto idx = td::Random::fast(1, static_cast(get_sources_cnt()) - 1); + if (idx == static_cast(local_idx_)) { + idx = 0; + } + // idx is a random number in [0, get_sources_cnt-1] not equal to local_idx + CatChainReceiverSource *S = get_source(idx); CHECK(S != nullptr); if (!S->blamed()) { synchronize_with(S); @@ -1117,6 +1048,23 @@ static void destroy_db(const std::string& name, td::uint32 attempt) { } } +void CatChainReceiverImpl::on_found_fork_proof(td::uint32 source_id, td::BufferSlice data) { + if (blame_processed_[source_id]) { + add_block(std::move(data), std::vector()); + } else { + pending_fork_proofs_[source_id] = std::move(data); + } +} + +void CatChainReceiverImpl::on_blame_processed(td::uint32 source_id) { + blame_processed_[source_id] = true; + auto it = pending_fork_proofs_.find(source_id); + if (it != pending_fork_proofs_.end()) { + add_block(std::move(it->second), std::vector()); + pending_fork_proofs_.erase(it); + } +} + void CatChainReceiverImpl::destroy() { auto name = db_root_ + "/catchainreceiver" + db_suffix_ + td::base64url_encode(as_slice(incarnation_)); delay_action([name]() { destroy_db(name, 0); }, td::Timestamp::in(DESTROY_DB_DELAY)); diff --git a/catchain/catchain-receiver.h b/catchain/catchain-receiver.h index 2c959fbc2..75c873511 100644 --- a/catchain/catchain-receiver.h +++ b/catchain/catchain-receiver.h @@ -56,7 +56,7 @@ class CatChainReceiver : public CatChainReceiverInterface { virtual void run_block(CatChainReceivedBlock *block) = 0; virtual void deliver_block(CatChainReceivedBlock *block) = 0; virtual td::uint32 add_fork() = 0; - virtual void add_prepared_event(td::BufferSlice data) = 0; + virtual void on_found_fork_proof(td::uint32 source_id, td::BufferSlice data) = 0; virtual void on_blame(td::uint32 source_id) = 0; virtual const CatChainOptions &opts() const = 0; diff --git a/catchain/catchain-receiver.hpp b/catchain/catchain-receiver.hpp index 5da4001c1..5c4d37648 100644 --- a/catchain/catchain-receiver.hpp +++ b/catchain/catchain-receiver.hpp @@ -39,9 +39,6 @@ class CatChainReceiverImpl final : public CatChainReceiver { return PrintId{incarnation_, local_id_}; } - void add_prepared_event(td::BufferSlice data) override { - add_block(std::move(data), std::vector()); - } CatChainSessionId get_incarnation() const override { return incarnation_; } @@ -73,15 +70,10 @@ class CatChainReceiverImpl final : public CatChainReceiver { void receive_query_from_overlay(adnl::AdnlNodeIdShort src, td::BufferSlice data, td::Promise promise); void process_query(adnl::AdnlNodeIdShort src, ton_api::catchain_getBlock query, td::Promise promise); - void process_query(adnl::AdnlNodeIdShort src, ton_api::catchain_getBlocks query, - td::Promise promise); - void process_query(adnl::AdnlNodeIdShort src, ton_api::catchain_getBlockHistory query, - td::Promise promise); void process_query(adnl::AdnlNodeIdShort src, ton_api::catchain_getDifference query, td::Promise promise); template void process_query(adnl::AdnlNodeIdShort src, const T &query, td::Promise promise) { - //LOG(WARNING) << this << ": unknown query from " << src; callback_->on_custom_query(get_source_by_adnl_id(src)->get_hash(), serialize_tl_object(&query, true), std::move(promise)); } @@ -89,7 +81,6 @@ class CatChainReceiverImpl final : public CatChainReceiver { void receive_block(adnl::AdnlNodeIdShort src, tl_object_ptr block, td::BufferSlice payload); void receive_block_answer(adnl::AdnlNodeIdShort src, td::BufferSlice); - //void send_block(const PublicKeyHash &src, tl_object_ptr block, td::BufferSlice payload); CatChainReceivedBlock *create_block(tl_object_ptr block, td::SharedSlice payload) override; CatChainReceivedBlock *create_block(tl_object_ptr block) override; @@ -117,6 +108,8 @@ class CatChainReceiverImpl final : public CatChainReceiver { void on_blame(td::uint32 src) override { callback_->blame(src); } + void on_found_fork_proof(td::uint32 source_id, td::BufferSlice data) override; + void on_blame_processed(td::uint32 source_id) override; const CatChainOptions &opts() const override { return opts_; } @@ -204,9 +197,6 @@ class CatChainReceiverImpl final : public CatChainReceiver { std::vector neighbours_; - //std::queue> events_; - //std::queue raw_events_; - td::actor::ActorId keyring_; td::actor::ActorId adnl_; td::actor::ActorId overlay_manager_; @@ -231,6 +221,9 @@ class CatChainReceiverImpl final : public CatChainReceiver { bool started_{false}; std::list to_run_; + + std::vector blame_processed_; + std::map pending_fork_proofs_; }; } // namespace catchain diff --git a/catchain/catchain.cpp b/catchain/catchain.cpp index 001840fd9..53ae16b52 100644 --- a/catchain/catchain.cpp +++ b/catchain/catchain.cpp @@ -108,11 +108,12 @@ void CatChainImpl::need_new_block(td::Timestamp t) { if (!receiver_started_) { return; } - if (!force_process_) { + if (!force_process_ || !active_process_) { VLOG(CATCHAIN_INFO) << this << ": forcing creation of new block"; } - force_process_ = true; - if (!active_process_) { + if (active_process_) { + force_process_ = true; + } else { alarm_timestamp().relax(t); } } @@ -197,6 +198,14 @@ void CatChainImpl::on_blame(td::uint32 src_id) { } } } + for (td::uint32 i = 0; i < process_deps_.size(); ++i) { + if (blocks_[process_deps_[i]]->source() == src_id) { + process_deps_[i] = process_deps_.back(); + process_deps_.pop_back(); + --i; + } + } + td::actor::send_closure(receiver_, &CatChainReceiverInterface::on_blame_processed, src_id); } void CatChainImpl::on_custom_query(const PublicKeyHash &src, td::BufferSlice data, td::Promise promise) { diff --git a/crypto/block/block.tlb b/crypto/block/block.tlb index 25e50a232..6ab035089 100644 --- a/crypto/block/block.tlb +++ b/crypto/block/block.tlb @@ -783,8 +783,8 @@ jetton_bridge_params_v0#00 bridge_address:bits256 oracles_address:bits256 oracle jetton_bridge_params_v1#01 bridge_address:bits256 oracles_address:bits256 oracles:(HashmapE 256 uint256) state_flags:uint8 prices:^JettonBridgePrices external_chain_address:bits256 = JettonBridgeParams; _ JettonBridgeParams = ConfigParam 79; // ETH->TON token bridge -_ JettonBridgeParams = ConfigParam 80; // BNB->TON token bridge -_ JettonBridgeParams = ConfigParam 81; // Polygon->TON token bridge +_ JettonBridgeParams = ConfigParam 81; // BNB->TON token bridge +_ JettonBridgeParams = ConfigParam 82; // Polygon->TON token bridge // diff --git a/crypto/fift/lib/Asm.fif b/crypto/fift/lib/Asm.fif index 85045f290..64b850096 100644 --- a/crypto/fift/lib/Asm.fif +++ b/crypto/fift/lib/Asm.fif @@ -2,7 +2,7 @@ library TVM_Asm // simple TVM Assembler namespace Asm Asm definitions -"0.4.3" constant asm-fif-version +"0.4.4" constant asm-fif-version variable @atend variable @was-split @@ -578,7 +578,7 @@ x{CF3F} @Defop BCHKBITREFSQ x{CF40} @Defop STZEROES x{CF41} @Defop STONES x{CF42} @Defop STSAME -{ tuck sbitrefs swap 15 + swap @havebitrefs not +{ tuck sbitrefs swap 22 + swap @havebitrefs not { swap PUSHSLICE STSLICER } { over sbitrefs 2dup 57 3 2x<= { rot x{CFC_} s, swap 2 u, over 6 + 3 >> tuck 3 u, 3 roll s, @@ -1194,6 +1194,11 @@ x{FA01} @Defop LDVARINT16 x{FA02} dup @Defop STGRAMS @Defop STVARUINT16 x{FA03} @Defop STVARINT16 +x{FA04} @Defop LDVARUINT32 // (s -- x s') +x{FA05} @Defop LDVARINT32 // (s -- x s') +x{FA06} @Defop STVARUINT32 // (b x -- b') +x{FA07} @Defop STVARINT32 // (b x -- b') + x{FA40} @Defop LDMSGADDR x{FA41} @Defop LDMSGADDRQ x{FA42} @Defop PARSEMSGADDR diff --git a/crypto/fift/words.cpp b/crypto/fift/words.cpp index 964b43288..b5e6dbd69 100644 --- a/crypto/fift/words.cpp +++ b/crypto/fift/words.cpp @@ -235,7 +235,7 @@ void interpret_cmp(vm::Stack& stack, const char opt[3]) { } void interpret_sgn(vm::Stack& stack, const char opt[3]) { - auto x = stack.pop_int(); + auto x = stack.pop_int_finite(); int r = x->sgn(); assert((unsigned)(r + 1) <= 2); stack.push_smallint(((const signed char*)opt)[r + 1]); diff --git a/crypto/func/abscode.cpp b/crypto/func/abscode.cpp index 933edc3d9..f1ffcfa49 100644 --- a/crypto/func/abscode.cpp +++ b/crypto/func/abscode.cpp @@ -38,6 +38,9 @@ TmpVar::TmpVar(var_idx_t _idx, int _cls, TypeExpr* _type, SymDef* sym, const Src if (!_type) { v_type = TypeExpr::new_hole(); } + if (cls == _Named) { + undefined = true; + } } void TmpVar::set_location(const SrcLocation& loc) { diff --git a/crypto/func/analyzer.cpp b/crypto/func/analyzer.cpp index 989e9ebe9..ec6931af0 100644 --- a/crypto/func/analyzer.cpp +++ b/crypto/func/analyzer.cpp @@ -42,14 +42,14 @@ int CodeBlob::split_vars(bool strict) { } std::vector comp_types; int k = var.v_type->extract_components(comp_types); - assert(k <= 254 && n <= 0x7fff00); - assert((unsigned)k == comp_types.size()); + func_assert(k <= 254 && n <= 0x7fff00); + func_assert((unsigned)k == comp_types.size()); if (k != 1) { var.coord = ~((n << 8) + k); for (int i = 0; i < k; i++) { auto v = create_var(vars[j].cls, comp_types[i], 0, vars[j].where.get()); - assert(v == n + i); - assert(vars[v].idx == v); + func_assert(v == n + i); + func_assert(vars[v].idx == v); vars[v].name = vars[j].name; vars[v].coord = ((int)j << 8) + i + 1; } @@ -75,9 +75,9 @@ bool CodeBlob::compute_used_code_vars() { } bool CodeBlob::compute_used_code_vars(std::unique_ptr& ops_ptr, const VarDescrList& var_info, bool edit) const { - assert(ops_ptr); + func_assert(ops_ptr); if (!ops_ptr->next) { - assert(ops_ptr->cl == Op::_Nop); + func_assert(ops_ptr->cl == Op::_Nop); return ops_ptr->set_var_info(var_info); } return compute_used_code_vars(ops_ptr->next, var_info, edit) | ops_ptr->compute_used_vars(*this, edit); @@ -346,7 +346,7 @@ bool Op::std_compute_used_vars(bool disabled) { } bool Op::compute_used_vars(const CodeBlob& code, bool edit) { - assert(next); + func_assert(next); const VarDescrList& next_var_info = next->var_info; if (cl == _Nop) { return set_var_info_except(next_var_info, left); @@ -379,7 +379,7 @@ bool Op::compute_used_vars(const CodeBlob& code, bool edit) { case _Let: { // left = right std::size_t cnt = next_var_info.count_used(left); - assert(left.size() == right.size()); + func_assert(left.size() == right.size()); auto l_it = left.cbegin(), r_it = right.cbegin(); VarDescrList new_var_info{next_var_info}; new_var_info -= left; @@ -500,7 +500,7 @@ bool Op::compute_used_vars(const CodeBlob& code, bool edit) { } changes = (new_var_info.size() == n); } while (changes <= edit); - assert(left.size() == 1); + func_assert(left.size() == 1); bool last = new_var_info.count_used(left) == 0; new_var_info += left; if (last) { @@ -655,7 +655,7 @@ bool prune_unreachable(std::unique_ptr& ops) { ops = std::move(op.block0); return false; } - reach = true; + reach = (op.cl != Op::_Again); break; } case Op::_TryCatch: { @@ -684,7 +684,7 @@ void CodeBlob::prune_unreachable_code() { void CodeBlob::fwd_analyze() { VarDescrList values; - assert(ops && ops->cl == Op::_Import); + func_assert(ops && ops->cl == Op::_Import); for (var_idx_t i : ops->left) { values += i; if (vars[i].v_type->is_int()) { @@ -765,7 +765,7 @@ VarDescrList Op::fwd_analyze(VarDescrList values) { break; case _Let: { std::vector old_val; - assert(left.size() == right.size()); + func_assert(left.size() == right.size()); for (std::size_t i = 0; i < right.size(); i++) { const VarDescr* ov = values[right[i]]; if (!ov && verbosity >= 5) { @@ -780,7 +780,7 @@ VarDescrList Op::fwd_analyze(VarDescrList values) { } std::cerr << std::endl; } - // assert(ov); + // func_assert(ov); if (ov) { old_val.push_back(*ov); } else { @@ -894,7 +894,7 @@ bool Op::mark_noreturn() { return set_noreturn((block0->mark_noreturn() & (block1 && block1->mark_noreturn())) | next->mark_noreturn()); case _Again: block0->mark_noreturn(); - return set_noreturn(false); + return set_noreturn(true); case _Until: return set_noreturn(block0->mark_noreturn() | next->mark_noreturn()); case _While: diff --git a/crypto/func/asmops.cpp b/crypto/func/asmops.cpp index ccc409d39..71ee58f68 100644 --- a/crypto/func/asmops.cpp +++ b/crypto/func/asmops.cpp @@ -166,7 +166,7 @@ AsmOp AsmOp::UnTuple(int a) { AsmOp AsmOp::IntConst(td::RefInt256 x) { if (x->signed_fits_bits(8)) { - return AsmOp::Const(dec_string(std::move(x)) + " PUSHINT", x); + return AsmOp::Const(dec_string(x) + " PUSHINT", x); } if (!x->is_valid()) { return AsmOp::Const("PUSHNAN", x); @@ -184,9 +184,9 @@ AsmOp AsmOp::IntConst(td::RefInt256 x) { return AsmOp::Const(k, "PUSHNEGPOW2", x); } if (!x->mod_pow2_short(23)) { - return AsmOp::Const(dec_string(std::move(x)) + " PUSHINTX", x); + return AsmOp::Const(dec_string(x) + " PUSHINTX", x); } - return AsmOp::Const(dec_string(std::move(x)) + " PUSHINT", x); + return AsmOp::Const(dec_string(x) + " PUSHINT", x); } AsmOp AsmOp::BoolConst(bool f) { diff --git a/crypto/func/auto-tests/legacy_tester.js b/crypto/func/auto-tests/legacy_tester.js new file mode 100644 index 000000000..57092d683 --- /dev/null +++ b/crypto/func/auto-tests/legacy_tester.js @@ -0,0 +1,27 @@ +const fs = require('fs/promises'); +const { compileWasm, compileFile } = require('./wasm_tests_common'); + +async function main() { + const tests = JSON.parse((await fs.readFile('../legacy_tests.json')).toString('utf-8')) + + for (const [filename, hashstr] of tests) { + if (filename.includes('storage-provider')) continue; + + const mod = await compileWasm() + + const response = await compileFile(mod, filename); + + if (response.status !== 'ok') { + console.error(response); + throw new Error('Could not compile ' + filename); + } + + if (BigInt('0x' + response.codeHashHex) !== BigInt(hashstr)) { + throw new Error('Compilation result is different for ' + filename); + } + + console.log(filename, 'ok'); + } +} + +main() \ No newline at end of file diff --git a/crypto/func/auto-tests/legacy_tests.json b/crypto/func/auto-tests/legacy_tests.json new file mode 100644 index 000000000..61a433bb7 --- /dev/null +++ b/crypto/func/auto-tests/legacy_tests.json @@ -0,0 +1 @@ +[["elector/elector-code.fc", "115226404411715505328583639896096915745686314074575650766750648324043316883483"], ["config/config-code.fc", "10913070768607625342121305745084703121685937915388357634624451844356456145601"], ["eth-bridge-multisig/multisig-code.fc", "101509909129354488841890823627011033360100627957439967918234053299675481277954"], ["bsc-bridge-collector/votes-collector.fc", "62190447221288642706570413295807615918589884489514159926097051017036969900417"], ["uni-lock-wallet/uni-lockup-wallet.fc", "61959738324779104851267145467044677651344601417998258530238254441977103654381"], ["nft-collection/nft-collection-editable.fc", "45561997735512210616567774035540357815786262097548276229169737015839077731274"], ["dns-collection/nft-collection.fc", "107999822699841936063083742021519765435859194241091312445235370766165379261859"], ["tele-nft-item/nft-item.fc", "69777543125381987786450436977742010705076866061362104025338034583422166453344"], ["storage/storage-contract.fc", "91377830060355733016937375216020277778264560226873154627574229667513068328151"], ["storage/storage-provider.fc", "13618336676213331164384407184540461509022654507176709588621016553953760588122"], ["nominator-pool/pool.fc", "69767057279163099864792356875696330339149706521019810113334238732928422055375"], ["jetton-minter/jetton-minter.fc", "9028309926287301331466371999814928201427184114165428257502393474125007156494"], ["gg-marketplace/nft-marketplace-v2.fc", "92199806964112524639740773542356508485601908152150843819273107618799016205930"], ["jetton-wallet/jetton-wallet.fc", "86251125787443633057458168028617933212663498001665054651523310772884328206542"], ["whales-nominators/nominators.fc", "8941364499854379927692172316865293429893094891593442801401542636695127885153"], ["tact-examples/treasure_Treasure.code.fc", "13962538639825790677138656603323869918938565499584297120566680287245364723897"], ["tact-examples/jetton_SampleJetton.code.fc", "94076762218493729104783735200107713211245710256802265203823917715299139499110"], ["tact-examples/jetton_JettonDefaultWallet.code.fc", "29421313492520031238091587108198906058157443241743283101866538036369069620563"], ["tact-examples/maps_MapTestContract.code.fc", "22556550222249123835909180266811414538971143565993192846012583552876721649744"]] \ No newline at end of file diff --git a/crypto/func/auto-tests/run_tests.js b/crypto/func/auto-tests/run_tests.js new file mode 100644 index 000000000..f8e6c6a78 --- /dev/null +++ b/crypto/func/auto-tests/run_tests.js @@ -0,0 +1,77 @@ +const fs = require('fs/promises'); +const os = require('os'); +const path = require('path'); +const { compileWasm, compileFile } = require('./wasm_tests_common'); +const { execSync } = require('child_process'); + +async function main() { + const compiledPath = path.join(os.tmpdir(), 'compiled.fif'); + const runnerPath = path.join(os.tmpdir(), 'runner.fif'); + + const tests = (await fs.readdir('.')).filter(f => f.endsWith('.fc')).sort(); + + const mathChars = '0x123456789()+-*/<>'.split('') + + for (const testFile of tests) { + const mod = await compileWasm() + + const result = await compileFile(mod, testFile) + + if (result.status !== 'ok') { + console.error(result); + throw new Error('Could not compile ' + filename); + } + + const fileLines = (await fs.readFile(testFile)).toString('utf-8').split('\n'); + + const testCases = []; + + for (const line of fileLines) { + const parts = line.split('|').map(c => c.trim()); + + if (parts.length !== 4 || parts[0] !== 'TESTCASE') continue; + + const processedInputs = []; + + for (const input of parts[2].split(' ')) { + if (input.includes('x{')) { + processedInputs.push(input); + continue; + } + + if (input.length === 0) { + continue + } + + const replacedInput = input.split('').filter(c => mathChars.includes(c)).join('').replace('//', '/').replace(/([0-9a-f])($|[^0-9a-fx])/gmi, '$1n$2') + + processedInputs.push(eval(replacedInput).toString()); + } + + testCases.push([parts[1], processedInputs.join(' '), parts[3]]); + } + + await fs.writeFile(compiledPath, '"Asm.fif" include\n' + JSON.parse('"' + result.fiftCode + '"')); + await fs.writeFile(runnerPath, `"${compiledPath}" include `${t[1]} ${t[0]} code 1 runvmx abort"exitcode is not 0" .s cr { drop } depth 1- times`).join('\n')}`) + + const fiftResult = execSync(`${process.env.FIFT_EXECUTABLE || 'fift'} -I ${process.env.FIFT_LIBS} /tmp/runner.fif`, { + stdio: ['pipe', 'pipe', 'ignore'] + }).toString('utf-8') + + const testResults = fiftResult.split('\n').map(s => s.trim()).filter(s => s.length > 0) + + if (testResults.length !== testCases.length) { + throw new Error(`Got ${testResults.length} results but there are ${testCases.length} cases`) + } + + for (let i = 0; i < testResults.length; i++) { + if (testResults[i] !== testCases[i][2]) { + throw new Error(`Unequal result ${testResults[i]} and case ${testCases[i][2]}`) + } + } + + console.log(testFile, 'ok') + } +} + +main() \ No newline at end of file diff --git a/crypto/func/auto-tests/tests/try-func.fc b/crypto/func/auto-tests/tests/try-func.fc new file mode 100644 index 000000000..5678965ef --- /dev/null +++ b/crypto/func/auto-tests/tests/try-func.fc @@ -0,0 +1,109 @@ +int foo(int x) method_id(11) { + try { + if (x == 7) { + throw(44); + } + return x; + } catch (_, _) { + return 2; + } +} + +int foo_inline(int x) inline method_id(12) { + try { + if (x == 7) { + throw(44); + } + return x; + } catch (_, _) { + return 2; + } +} + +int foo_inlineref(int x) inline_ref method_id(13) { + try { + if (x == 7) { + throw(44); + } + return x; + } catch (_, _) { + return 2; + } +} + +int test(int x, int y, int z) method_id(1) { + y = foo(y); + return x * 100 + y * 10 + z; +} + +int test_inline(int x, int y, int z) method_id(2) { + y = foo_inline(y); + return x * 100 + y * 10 + z; +} + +int test_inlineref(int x, int y, int z) method_id(3) { + y = foo_inlineref(y); + return x * 100 + y * 10 + z; +} + +int foo_inline_big( + int x1, int x2, int x3, int x4, int x5, int x6, int x7, int x8, int x9, int x10, + int x11, int x12, int x13, int x14, int x15, int x16, int x17, int x18, int x19, int x20 +) inline method_id(14) { + try { + if (x1 == 7) { + throw(44); + } + return x1 + x2 + x3 + x4 + x5 + x6 + x7 + x8 + x9 + x10 + x11 + x12 + x13 + x14 + x15 + x16 + x17 + x18 + x19 + x20; + } catch (_, _) { + return 1; + } +} + +int test_inline_big(int x, int y, int z) method_id(4) { + y = foo_inline_big( + y, y + 1, y + 2, y + 3, y + 4, y + 5, y + 6, y + 7, y + 8, y + 9, + y + 10, y + 11, y + 12, y + 13, y + 14, y + 15, y + 16, y + 17, y + 18, y + 19); + return x * 1000000 + y * 1000 + z; +} + +int foo_big( + int x1, int x2, int x3, int x4, int x5, int x6, int x7, int x8, int x9, int x10, + int x11, int x12, int x13, int x14, int x15, int x16, int x17, int x18, int x19, int x20 +) method_id(15) { + try { + if (x1 == 7) { + throw(44); + } + return x1 + x2 + x3 + x4 + x5 + x6 + x7 + x8 + x9 + x10 + x11 + x12 + x13 + x14 + x15 + x16 + x17 + x18 + x19 + x20; + } catch (_, _) { + return 1; + } +} + +int test_big(int x, int y, int z) method_id(5) { + y = foo_big( + y, y + 1, y + 2, y + 3, y + 4, y + 5, y + 6, y + 7, y + 8, y + 9, + y + 10, y + 11, y + 12, y + 13, y + 14, y + 15, y + 16, y + 17, y + 18, y + 19); + return x * 1000000 + y * 1000 + z; +} + +() main() { +} + +{- + method_id | in | out +TESTCASE | 1 | 1 2 3 | 123 +TESTCASE | 1 | 3 8 9 | 389 +TESTCASE | 1 | 3 7 9 | 329 +TESTCASE | 2 | 1 2 3 | 123 +TESTCASE | 2 | 3 8 9 | 389 +TESTCASE | 2 | 3 7 9 | 329 +TESTCASE | 3 | 1 2 3 | 123 +TESTCASE | 3 | 3 8 9 | 389 +TESTCASE | 3 | 3 7 9 | 329 +TESTCASE | 4 | 4 8 9 | 4350009 +TESTCASE | 4 | 4 7 9 | 4001009 +TESTCASE | 5 | 4 8 9 | 4350009 +TESTCASE | 5 | 4 7 9 | 4001009 +-} diff --git a/crypto/func/auto-tests/wasm_tests_common.js b/crypto/func/auto-tests/wasm_tests_common.js new file mode 100644 index 000000000..84d8e0d57 --- /dev/null +++ b/crypto/func/auto-tests/wasm_tests_common.js @@ -0,0 +1,63 @@ +const fsSync = require('fs'); + +const copyToCString = (mod, str) => { + const len = mod.lengthBytesUTF8(str) + 1; + const ptr = mod._malloc(len); + mod.stringToUTF8(str, ptr, len); + return ptr; +}; + +const copyToCStringPtr = (mod, str, ptr) => { + const allocated = copyToCString(mod, str); + mod.setValue(ptr, allocated, '*'); + return allocated; +}; + +const copyFromCString = (mod, ptr) => { + return mod.UTF8ToString(ptr); +}; + +async function compileFile(mod, filename) { + const callbackPtr = mod.addFunction((_kind, _data, contents, error) => { + const kind = copyFromCString(mod, _kind); + const data = copyFromCString(mod, _data); + if (kind === 'realpath') { + copyToCStringPtr(mod, fsSync.realpathSync(data), contents); + } else if (kind === 'source') { + const path = fsSync.realpathSync(data); + try { + copyToCStringPtr(mod, fsSync.readFileSync(path).toString('utf-8'), contents); + } catch (err) { + copyToCStringPtr(mod, e.message, error); + } + } else { + copyToCStringPtr(mod, 'Unknown callback kind ' + kind, error); + } + }, 'viiii'); + + const config = { + optLevel: 2, + sources: [filename] + }; + + const configPtr = copyToCString(mod, JSON.stringify(config)); + + const responsePtr = mod._func_compile(configPtr, callbackPtr); + + return JSON.parse(copyFromCString(mod, responsePtr)); +} + +const wasmModule = require(process.env.FUNCFIFTLIB_MODULE) + +const wasmBinary = new Uint8Array(fsSync.readFileSync(process.env.FUNCFIFTLIB_WASM)) + +async function compileWasm() { + const mod = await wasmModule({ wasmBinary }) + + return mod +} + +module.exports = { + compileFile, + compileWasm +} diff --git a/crypto/func/builtins.cpp b/crypto/func/builtins.cpp index 296910347..9de673fd7 100644 --- a/crypto/func/builtins.cpp +++ b/crypto/func/builtins.cpp @@ -429,7 +429,7 @@ AsmOp push_const(td::RefInt256 x) { } AsmOp compile_add(std::vector& res, std::vector& args, const SrcLocation& where) { - assert(res.size() == 1 && args.size() == 2); + func_assert(res.size() == 1 && args.size() == 2); VarDescr &r = res[0], &x = args[0], &y = args[1]; if (x.is_int_const() && y.is_int_const()) { r.set_const(x.int_const + y.int_const); @@ -471,7 +471,7 @@ AsmOp compile_add(std::vector& res, std::vector& args, const } AsmOp compile_sub(std::vector& res, std::vector& args, const SrcLocation& where) { - assert(res.size() == 1 && args.size() == 2); + func_assert(res.size() == 1 && args.size() == 2); VarDescr &r = res[0], &x = args[0], &y = args[1]; if (x.is_int_const() && y.is_int_const()) { r.set_const(x.int_const - y.int_const); @@ -504,7 +504,7 @@ AsmOp compile_sub(std::vector& res, std::vector& args, const } AsmOp compile_negate(std::vector& res, std::vector& args, const SrcLocation& where) { - assert(res.size() == 1 && args.size() == 1); + func_assert(res.size() == 1 && args.size() == 1); VarDescr &r = res[0], &x = args[0]; if (x.is_int_const()) { r.set_const(-x.int_const); @@ -519,7 +519,7 @@ AsmOp compile_negate(std::vector& res, std::vector& args, co } AsmOp compile_and(std::vector& res, std::vector& args, const SrcLocation& where) { - assert(res.size() == 1 && args.size() == 2); + func_assert(res.size() == 1 && args.size() == 2); VarDescr &r = res[0], &x = args[0], &y = args[1]; if (x.is_int_const() && y.is_int_const()) { r.set_const(x.int_const & y.int_const); @@ -532,7 +532,7 @@ AsmOp compile_and(std::vector& res, std::vector& args, const } AsmOp compile_or(std::vector& res, std::vector& args, const SrcLocation& where) { - assert(res.size() == 1 && args.size() == 2); + func_assert(res.size() == 1 && args.size() == 2); VarDescr &r = res[0], &x = args[0], &y = args[1]; if (x.is_int_const() && y.is_int_const()) { r.set_const(x.int_const | y.int_const); @@ -545,7 +545,7 @@ AsmOp compile_or(std::vector& res, std::vector& args, const } AsmOp compile_xor(std::vector& res, std::vector& args, const SrcLocation& where) { - assert(res.size() == 1 && args.size() == 2); + func_assert(res.size() == 1 && args.size() == 2); VarDescr &r = res[0], &x = args[0], &y = args[1]; if (x.is_int_const() && y.is_int_const()) { r.set_const(x.int_const ^ y.int_const); @@ -558,7 +558,7 @@ AsmOp compile_xor(std::vector& res, std::vector& args, const } AsmOp compile_not(std::vector& res, std::vector& args, const SrcLocation& where) { - assert(res.size() == 1 && args.size() == 1); + func_assert(res.size() == 1 && args.size() == 1); VarDescr &r = res[0], &x = args[0]; if (x.is_int_const()) { r.set_const(~x.int_const); @@ -638,12 +638,12 @@ AsmOp compile_mul_internal(VarDescr& r, VarDescr& x, VarDescr& y, const SrcLocat } AsmOp compile_mul(std::vector& res, std::vector& args, const SrcLocation& where) { - assert(res.size() == 1 && args.size() == 2); + func_assert(res.size() == 1 && args.size() == 2); return compile_mul_internal(res[0], args[0], args[1], where); } AsmOp compile_lshift(std::vector& res, std::vector& args, const SrcLocation& where) { - assert(res.size() == 1 && args.size() == 2); + func_assert(res.size() == 1 && args.size() == 2); VarDescr &r = res[0], &x = args[0], &y = args[1]; if (y.is_int_const()) { auto yv = y.int_const->to_long(); @@ -686,7 +686,7 @@ AsmOp compile_lshift(std::vector& res, std::vector& args, co AsmOp compile_rshift(std::vector& res, std::vector& args, const SrcLocation& where, int round_mode) { - assert(res.size() == 1 && args.size() == 2); + func_assert(res.size() == 1 && args.size() == 2); VarDescr &r = res[0], &x = args[0], &y = args[1]; if (y.is_int_const()) { auto yv = y.int_const->to_long(); @@ -755,13 +755,13 @@ AsmOp compile_div_internal(VarDescr& r, VarDescr& x, VarDescr& y, const SrcLocat } AsmOp compile_div(std::vector& res, std::vector& args, const SrcLocation& where, int round_mode) { - assert(res.size() == 1 && args.size() == 2); + func_assert(res.size() == 1 && args.size() == 2); return compile_div_internal(res[0], args[0], args[1], where, round_mode); } AsmOp compile_mod(std::vector& res, std::vector& args, const src::SrcLocation& where, int round_mode) { - assert(res.size() == 1 && args.size() == 2); + func_assert(res.size() == 1 && args.size() == 2); VarDescr &r = res[0], &x = args[0], &y = args[1]; if (x.is_int_const() && y.is_int_const()) { r.set_const(mod(x.int_const, y.int_const, round_mode)); @@ -802,7 +802,7 @@ AsmOp compile_mod(std::vector& res, std::vector& args, const AsmOp compile_muldiv(std::vector& res, std::vector& args, const SrcLocation& where, int round_mode) { - assert(res.size() == 1 && args.size() == 3); + func_assert(res.size() == 1 && args.size() == 3); VarDescr &r = res[0], &x = args[0], &y = args[1], &z = args[2]; if (x.is_int_const() && y.is_int_const() && z.is_int_const()) { r.set_const(muldiv(x.int_const, y.int_const, z.int_const, round_mode)); @@ -923,8 +923,8 @@ int compute_compare(const VarDescr& x, const VarDescr& y, int mode) { } AsmOp compile_cmp_int(std::vector& res, std::vector& args, int mode) { - assert(mode >= 1 && mode <= 7); - assert(res.size() == 1 && args.size() == 2); + func_assert(mode >= 1 && mode <= 7); + func_assert(res.size() == 1 && args.size() == 2); VarDescr &r = res[0], &x = args[0], &y = args[1]; if (x.is_int_const() && y.is_int_const()) { int v = compute_compare(x.int_const, y.int_const, mode); @@ -935,7 +935,7 @@ AsmOp compile_cmp_int(std::vector& res, std::vector& args, i } int v = compute_compare(x, y, mode); // std::cerr << "compute_compare(" << x << ", " << y << ", " << mode << ") = " << v << std::endl; - assert(v); + func_assert(v); if (!(v & (v - 1))) { r.set_const(v - (v >> 2) - 2); x.unused(); @@ -971,7 +971,7 @@ AsmOp compile_cmp_int(std::vector& res, std::vector& args, i } AsmOp compile_throw(std::vector& res, std::vector& args, const SrcLocation&) { - assert(res.empty() && args.size() == 1); + func_assert(res.empty() && args.size() == 1); VarDescr& x = args[0]; if (x.is_int_const() && x.int_const->unsigned_fits_bits(11)) { x.unused(); @@ -982,7 +982,7 @@ AsmOp compile_throw(std::vector& res, std::vector& args, con } AsmOp compile_cond_throw(std::vector& res, std::vector& args, bool mode) { - assert(res.empty() && args.size() == 2); + func_assert(res.empty() && args.size() == 2); VarDescr &x = args[0], &y = args[1]; std::string suff = (mode ? "IF" : "IFNOT"); bool skip_cond = false; @@ -1003,7 +1003,7 @@ AsmOp compile_cond_throw(std::vector& res, std::vector& args } AsmOp compile_throw_arg(std::vector& res, std::vector& args, const SrcLocation&) { - assert(res.empty() && args.size() == 2); + func_assert(res.empty() && args.size() == 2); VarDescr &x = args[1]; if (x.is_int_const() && x.int_const->unsigned_fits_bits(11)) { x.unused(); @@ -1014,7 +1014,7 @@ AsmOp compile_throw_arg(std::vector& res, std::vector& args, } AsmOp compile_cond_throw_arg(std::vector& res, std::vector& args, bool mode) { - assert(res.empty() && args.size() == 3); + func_assert(res.empty() && args.size() == 3); VarDescr &x = args[1], &y = args[2]; std::string suff = (mode ? "IF" : "IFNOT"); bool skip_cond = false; @@ -1035,7 +1035,7 @@ AsmOp compile_cond_throw_arg(std::vector& res, std::vector& } AsmOp compile_bool_const(std::vector& res, std::vector& args, bool val) { - assert(res.size() == 1 && args.empty()); + func_assert(res.size() == 1 && args.empty()); VarDescr& r = res[0]; r.set_const(val ? -1 : 0); return AsmOp::Const(val ? "TRUE" : "FALSE"); @@ -1046,7 +1046,7 @@ AsmOp compile_bool_const(std::vector& res, std::vector& args // int preload_int(slice s, int len) asm "PLDIX"; // int preload_uint(slice s, int len) asm "PLDUX"; AsmOp compile_fetch_int(std::vector& res, std::vector& args, bool fetch, bool sgnd) { - assert(args.size() == 2 && res.size() == 1 + (unsigned)fetch); + func_assert(args.size() == 2 && res.size() == 1 + (unsigned)fetch); auto &y = args[1], &r = res.back(); r.val = (sgnd ? VarDescr::FiniteInt : VarDescr::FiniteUInt); int v = -1; @@ -1069,7 +1069,7 @@ AsmOp compile_fetch_int(std::vector& res, std::vector& args, // builder store_uint(builder b, int x, int len) asm(x b len) "STUX"; // builder store_int(builder b, int x, int len) asm(x b len) "STIX"; AsmOp compile_store_int(std::vector& res, std::vector& args, bool sgnd) { - assert(args.size() == 3 && res.size() == 1); + func_assert(args.size() == 3 && res.size() == 1); auto& z = args[2]; if (z.is_int_const() && z.int_const > 0 && z.int_const <= 256) { z.unused(); @@ -1079,7 +1079,7 @@ AsmOp compile_store_int(std::vector& res, std::vector& args, } AsmOp compile_fetch_slice(std::vector& res, std::vector& args, bool fetch) { - assert(args.size() == 2 && res.size() == 1 + (unsigned)fetch); + func_assert(args.size() == 2 && res.size() == 1 + (unsigned)fetch); auto& y = args[1]; int v = -1; if (y.is_int_const() && y.int_const > 0 && y.int_const <= 256) { @@ -1094,7 +1094,7 @@ AsmOp compile_fetch_slice(std::vector& res, std::vector& arg // _at(tuple t, int index) asm "INDEXVAR"; AsmOp compile_tuple_at(std::vector& res, std::vector& args, const SrcLocation&) { - assert(args.size() == 2 && res.size() == 1); + func_assert(args.size() == 2 && res.size() == 1); auto& y = args[1]; if (y.is_int_const() && y.int_const >= 0 && y.int_const < 16) { y.unused(); @@ -1105,7 +1105,7 @@ AsmOp compile_tuple_at(std::vector& res, std::vector& args, // int null?(X arg) AsmOp compile_is_null(std::vector& res, std::vector& args, const SrcLocation&) { - assert(args.size() == 1 && res.size() == 1); + func_assert(args.size() == 1 && res.size() == 1); auto &x = args[0], &r = res[0]; if (x.always_null() || x.always_not_null()) { x.unused(); @@ -1118,7 +1118,7 @@ AsmOp compile_is_null(std::vector& res, std::vector& args, c bool compile_run_method(AsmOpList& code, std::vector& res, std::vector& args, int n, bool has_value) { - assert(args.size() == (unsigned)n + 1 && res.size() == (unsigned)has_value); + func_assert(args.size() == (unsigned)n + 1 && res.size() == (unsigned)has_value); auto& x = args[0]; if (x.is_int_const() && x.int_const->unsigned_fits_bits(14)) { x.unused(); diff --git a/crypto/func/codegen.cpp b/crypto/func/codegen.cpp index 872909655..de45c8418 100644 --- a/crypto/func/codegen.cpp +++ b/crypto/func/codegen.cpp @@ -165,7 +165,7 @@ void Stack::push_new_const(var_idx_t idx, const_idx_t cidx) { void Stack::assign_var(var_idx_t new_idx, var_idx_t old_idx) { int i = find(old_idx); - assert(i >= 0 && "variable not found in stack"); + func_assert(i >= 0 && "variable not found in stack"); if (new_idx != old_idx) { at(i).first = new_idx; modified(); @@ -174,10 +174,10 @@ void Stack::assign_var(var_idx_t new_idx, var_idx_t old_idx) { void Stack::do_copy_var(var_idx_t new_idx, var_idx_t old_idx) { int i = find(old_idx); - assert(i >= 0 && "variable not found in stack"); + func_assert(i >= 0 && "variable not found in stack"); if (find(old_idx, i + 1) < 0) { issue_push(i); - assert(at(0).first == old_idx); + func_assert(at(0).first == old_idx); } assign_var(new_idx, old_idx); } @@ -199,21 +199,21 @@ void Stack::enforce_state(const StackLayout& req_stack) { j = 0; } issue_xchg(j, depth() - i - 1); - assert(s[i].first == x); + func_assert(s[i].first == x); } while (depth() > k) { issue_pop(0); } - assert(depth() == k); + func_assert(depth() == k); for (int i = 0; i < k; i++) { - assert(s[i].first == req_stack[i]); + func_assert(s[i].first == req_stack[i]); } } void Stack::merge_const(const Stack& req_stack) { - assert(s.size() == req_stack.s.size()); + func_assert(s.size() == req_stack.s.size()); for (std::size_t i = 0; i < s.size(); i++) { - assert(s[i].first == req_stack.s[i].first); + func_assert(s[i].first == req_stack.s[i].first); if (s[i].second != req_stack.s[i].second) { s[i].second = not_const; } @@ -251,15 +251,15 @@ void Stack::rearrange_top(const StackLayout& top, std::vector last) { if (last[i]) { // rearrange x to be at s(ss-1) issue_xchg(--ss, j); - assert(get(ss).first == x); + func_assert(get(ss).first == x); } else { // create a new copy of x issue_push(j); issue_xchg(0, ss); - assert(get(ss).first == x); + func_assert(get(ss).first == x); } } - assert(!ss); + func_assert(!ss); } void Stack::rearrange_top(var_idx_t top, bool last) { @@ -269,7 +269,7 @@ void Stack::rearrange_top(var_idx_t top, bool last) { } else { issue_push(i); } - assert(get(0).first == top); + func_assert(get(0).first == top); } bool Op::generate_code_step(Stack& stack) { @@ -300,7 +300,7 @@ bool Op::generate_code_step(Stack& stack) { stack.o << push_const(int_const); stack.push_new_const(left[0], cidx); } else { - assert(stack.at(i).second == cidx); + func_assert(stack.at(i).second == cidx); stack.do_copy_var(left[0], stack[i]); } return true; @@ -329,7 +329,7 @@ bool Op::generate_code_step(Stack& stack) { std::string name = sym::symbols.get_name(fun_ref->sym_idx); stack.o << AsmOp::Custom(name + " GETGLOB", 0, 1); if (left.size() != 1) { - assert(left.size() <= 15); + func_assert(left.size() <= 15); stack.o << AsmOp::UnTuple((int)left.size()); } for (auto i : left) { @@ -337,7 +337,7 @@ bool Op::generate_code_step(Stack& stack) { } return true; } else { - assert(left.size() == 1); + func_assert(left.size() == 1); auto p = next->var_info[left[0]]; if (!p || p->is_unused() || disabled()) { return true; @@ -349,10 +349,10 @@ bool Op::generate_code_step(Stack& stack) { // TODO: create and compile a true lambda instead of this (so that arg_order and ret_order would work correctly) std::vector args0, res; TypeExpr::remove_indirect(func->sym_type); - assert(func->get_type()->is_map()); + func_assert(func->get_type()->is_map()); auto wr = func->get_type()->args.at(0)->get_width(); auto wl = func->get_type()->args.at(1)->get_width(); - assert(wl >= 0 && wr >= 0); + func_assert(wl >= 0 && wr >= 0); for (int i = 0; i < wl; i++) { res.emplace_back(0); } @@ -370,7 +370,7 @@ bool Op::generate_code_step(Stack& stack) { return true; } case _Let: { - assert(left.size() == right.size()); + func_assert(left.size() == right.size()); int i = 0; std::vector active; active.reserve(left.size()); @@ -420,13 +420,13 @@ bool Op::generate_code_step(Stack& stack) { stack.rearrange_top(right, std::move(last)); stack.opt_show(); int k = (int)stack.depth() - (int)right.size(); - assert(k >= 0); + func_assert(k >= 0); if (cl == _Tuple) { stack.o << AsmOp::Tuple((int)right.size()); - assert(left.size() == 1); + func_assert(left.size() == 1); } else { stack.o << AsmOp::UnTuple((int)left.size()); - assert(right.size() == 1); + func_assert(right.size() == 1); } stack.s.resize(k); for (int i = 0; i < (int)left.size(); i++) { @@ -442,16 +442,16 @@ bool Op::generate_code_step(Stack& stack) { SymValFunc* func = (fun_ref ? dynamic_cast(fun_ref->value) : nullptr); auto arg_order = (func ? func->get_arg_order() : nullptr); auto ret_order = (func ? func->get_ret_order() : nullptr); - assert(!arg_order || arg_order->size() == right.size()); - assert(!ret_order || ret_order->size() == left.size()); + func_assert(!arg_order || arg_order->size() == right.size()); + func_assert(!ret_order || ret_order->size() == left.size()); std::vector right1; if (args.size()) { - assert(args.size() == right.size()); + func_assert(args.size() == right.size()); for (int i = 0; i < (int)right.size(); i++) { int j = arg_order ? arg_order->at(i) : i; const VarDescr& arg = args.at(j); if (!arg.is_unused()) { - assert(var_info[arg.idx] && !var_info[arg.idx]->is_unused()); + func_assert(var_info[arg.idx] && !var_info[arg.idx]->is_unused()); right1.push_back(arg.idx); } } @@ -469,17 +469,25 @@ bool Op::generate_code_step(Stack& stack) { stack.rearrange_top(right1, std::move(last)); stack.opt_show(); int k = (int)stack.depth() - (int)right1.size(); - assert(k >= 0); + func_assert(k >= 0); for (int i = 0; i < (int)right1.size(); i++) { if (stack.s[k + i].first != right1[i]) { std::cerr << stack.o; } - assert(stack.s[k + i].first == right1[i]); + func_assert(stack.s[k + i].first == right1[i]); } + auto exec_callxargs = [&](int args, int ret) { + if (args <= 15 && ret <= 15) { + stack.o << exec_arg2_op("CALLXARGS", args, ret, args + 1, ret); + } else { + func_assert(args <= 254 && ret <= 254); + stack.o << AsmOp::Const(PSTRING() << args << " PUSHINT"); + stack.o << AsmOp::Const(PSTRING() << ret << " PUSHINT"); + stack.o << AsmOp::Custom("CALLXVARARGS", args + 3, ret); + } + }; if (cl == _CallInd) { - // TODO: replace with exec_arg2_op() - stack.o << exec_arg2_op("CALLXARGS", (int)right.size() - 1, (int)left.size(), (int)right.size(), - (int)left.size()); + exec_callxargs((int)right.size() - 1, (int)left.size()); } else { auto func = dynamic_cast(fun_ref->value); if (func) { @@ -493,8 +501,14 @@ bool Op::generate_code_step(Stack& stack) { auto fv = dynamic_cast(fun_ref->value); std::string name = sym::symbols.get_name(fun_ref->sym_idx); bool is_inline = (fv && (fv->flags & 3)); - stack.o << AsmOp::Custom(name + (is_inline ? " INLINECALLDICT" : " CALLDICT"), (int)right.size(), - (int)left.size()); + if (is_inline) { + stack.o << AsmOp::Custom(name + " INLINECALLDICT", (int)right.size(), (int)left.size()); + } else if (fv && fv->code && fv->code->require_callxargs) { + stack.o << AsmOp::Custom(name + (" PREPAREDICT"), 0, 2); + exec_callxargs((int)right.size() + 1, (int)left.size()); + } else { + stack.o << AsmOp::Custom(name + " CALLDICT", (int)right.size(), (int)left.size()); + } } } stack.s.resize(k); @@ -505,7 +519,7 @@ bool Op::generate_code_step(Stack& stack) { return true; } case _SetGlob: { - assert(fun_ref && dynamic_cast(fun_ref->value)); + func_assert(fun_ref && dynamic_cast(fun_ref->value)); std::vector last; for (var_idx_t x : right) { last.push_back(var_info[x] && var_info[x]->is_last()); @@ -513,12 +527,12 @@ bool Op::generate_code_step(Stack& stack) { stack.rearrange_top(right, std::move(last)); stack.opt_show(); int k = (int)stack.depth() - (int)right.size(); - assert(k >= 0); + func_assert(k >= 0); for (int i = 0; i < (int)right.size(); i++) { if (stack.s[k + i].first != right[i]) { std::cerr << stack.o; } - assert(stack.s[k + i].first == right[i]); + func_assert(stack.s[k + i].first == right[i]); } if (right.size() > 1) { stack.o << AsmOp::Tuple((int)right.size()); @@ -539,7 +553,7 @@ bool Op::generate_code_step(Stack& stack) { } var_idx_t x = left[0]; stack.rearrange_top(x, var_info[x] && var_info[x]->is_last()); - assert(stack[0] == x); + func_assert(stack[0] == x); stack.opt_show(); stack.s.pop_back(); stack.modified(); @@ -651,7 +665,7 @@ bool Op::generate_code_step(Stack& stack) { var_idx_t x = left[0]; //stack.drop_vars_except(block0->var_info, x); stack.rearrange_top(x, var_info[x] && var_info[x]->is_last()); - assert(stack[0] == x); + func_assert(stack[0] == x); stack.opt_show(); stack.s.pop_back(); stack.modified(); @@ -877,12 +891,13 @@ void Op::generate_code_all(Stack& stack) { void CodeBlob::generate_code(AsmOpList& out, int mode) { Stack stack{out, mode}; - assert(ops && ops->cl == Op::_Import); + func_assert(ops && ops->cl == Op::_Import); + auto args = (int)ops->left.size(); for (var_idx_t x : ops->left) { stack.push_new_var(x); } ops->generate_code_all(stack); - stack.apply_wrappers(); + stack.apply_wrappers(require_callxargs && (mode & Stack::_InlineAny) ? args : -1); if (!(mode & Stack::_DisableOpt)) { optimize_code(out); } diff --git a/crypto/func/func.cpp b/crypto/func/func.cpp index 87bf23e98..39648c05a 100644 --- a/crypto/func/func.cpp +++ b/crypto/func/func.cpp @@ -72,7 +72,7 @@ td::Result fs_read_callback(ReadCallback::Kind kind, const char* qu void generate_output_func(SymDef* func_sym, std::ostream &outs, std::ostream &errs) { SymValCodeFunc* func_val = dynamic_cast(func_sym->value); - assert(func_val); + func_assert(func_val); std::string name = sym::symbols.get_name(func_sym->sym_idx); if (verbosity >= 2) { errs << "\n\n=========================\nfunction " << name << " : " << func_val->get_type() << std::endl; @@ -145,6 +145,9 @@ void generate_output_func(SymDef* func_sym, std::ostream &outs, std::ostream &er if (fv && (fv->flags & 1) && code.ops->noreturn()) { mode |= Stack::_InlineFunc; } + if (fv && (fv->flags & 3)) { + mode |= Stack::_InlineAny; + } code.generate_code(outs, mode, indent + 1); outs << std::string(indent * 2, ' ') << "}>\n"; if (verbosity >= 2) { @@ -163,7 +166,7 @@ int generate_output(std::ostream &outs, std::ostream &errs) { } for (SymDef* func_sym : glob_func) { SymValCodeFunc* func_val = dynamic_cast(func_sym->value); - assert(func_val); + func_assert(func_val); std::string name = sym::symbols.get_name(func_sym->sym_idx); outs << std::string(indent * 2, ' '); if (func_val->method_id.is_null()) { @@ -173,7 +176,7 @@ int generate_output(std::ostream &outs, std::ostream &errs) { } } for (SymDef* gvar_sym : glob_vars) { - assert(dynamic_cast(gvar_sym->value)); + func_assert(dynamic_cast(gvar_sym->value)); std::string name = sym::symbols.get_name(gvar_sym->sym_idx); outs << std::string(indent * 2, ' ') << "DECLGLOBVAR " << name << "\n"; } diff --git a/crypto/func/func.h b/crypto/func/func.h index 10039ffae..2b95bcbc5 100644 --- a/crypto/func/func.h +++ b/crypto/func/func.h @@ -33,6 +33,10 @@ #include "parser/symtable.h" #include "td/utils/Status.h" +#define func_assert(expr) \ + (bool(expr) ? void(0) \ + : throw src::Fatal(PSTRING() << "Assertion failed at " << __FILE__ << ":" << __LINE__ << ": " << #expr)) + namespace funC { extern int verbosity; @@ -41,7 +45,7 @@ extern std::string generated_from; constexpr int optimize_depth = 20; -const std::string func_version{"0.4.3"}; +const std::string func_version{"0.4.4"}; enum Keyword { _Eof = -1, @@ -159,6 +163,7 @@ struct TypeExpr { int minw, maxw; static constexpr int w_inf = 1023; std::vector args; + bool was_forall_var = false; TypeExpr(te_type _constr, int _val = 0) : constr(_constr), value(_val), minw(0), maxw(w_inf) { } TypeExpr(te_type _constr, int _val, int width) : constr(_constr), value(_val), minw(width), maxw(width) { @@ -265,7 +270,7 @@ struct TypeExpr { return new TypeExpr{te_ForAll, body, std::move(list)}; } static bool remove_indirect(TypeExpr*& te, TypeExpr* forbidden = nullptr); - static bool remove_forall(TypeExpr*& te); + static std::vector remove_forall(TypeExpr*& te); static bool remove_forall_in(TypeExpr*& te, TypeExpr* te2, const std::vector& new_vars); }; @@ -309,6 +314,7 @@ struct TmpVar { int coord; std::unique_ptr where; std::vector> on_modification; + bool undefined = false; TmpVar(var_idx_t _idx, int _cls, TypeExpr* _type = 0, SymDef* sym = 0, const SrcLocation* loc = 0); void show(std::ostream& os, int omit_idx = 0) const; void dump(std::ostream& os) const; @@ -693,6 +699,7 @@ struct CodeBlob { std::unique_ptr* cur_ops; std::stack*> cur_ops_stack; int flags = 0; + bool require_callxargs = false; CodeBlob(TypeExpr* ret = nullptr) : var_cnt(0), in_var_cnt(0), op_cnt(0), ret_type(ret), cur_ops(&ops) { } template @@ -1176,7 +1183,7 @@ struct AsmOpList { } template AsmOpList& add(Args&&... args) { - list_.emplace_back(std::forward(args)...); + append(AsmOp(std::forward(args)...)); adjust_last(); return *this; } @@ -1563,8 +1570,8 @@ struct Stack { AsmOpList& o; enum { _StkCmt = 1, _CptStkCmt = 2, _DisableOpt = 4, _DisableOut = 128, _Shown = 256, - _InlineFunc = 512, _NeedRetAlt = 1024, - _ModeSave = _InlineFunc | _NeedRetAlt, + _InlineFunc = 512, _NeedRetAlt = 1024, _InlineAny = 2048, + _ModeSave = _InlineFunc | _NeedRetAlt | _InlineAny, _Garbage = -0x10000 }; int mode; @@ -1611,7 +1618,7 @@ struct Stack { if (i > 255) { throw src::Fatal{"Too deep stack"}; } - assert(i >= 0 && i < depth() && "invalid stack reference"); + func_assert(i >= 0 && i < depth() && "invalid stack reference"); } void modified() { mode &= ~_Shown; @@ -1642,14 +1649,24 @@ struct Stack { bool operator==(const Stack& y) const & { return s == y.s; } - void apply_wrappers() { + void apply_wrappers(int callxargs_count) { + bool is_inline = mode & _InlineFunc; if (o.retalt_) { o.insert(0, "SAMEALTSAVE"); o.insert(0, "c2 SAVE"); - if (mode & _InlineFunc) { - o.indent_all(); - o.insert(0, "CONT:<{"); - o << "}>"; + } + if (callxargs_count != -1 || (is_inline && o.retalt_)) { + o.indent_all(); + o.insert(0, "CONT:<{"); + o << "}>"; + if (callxargs_count != -1) { + if (callxargs_count <= 15) { + o << AsmOp::Custom(PSTRING() << callxargs_count << " -1 CALLXARGS"); + } else { + func_assert(callxargs_count <= 254); + o << AsmOp::Custom(PSTRING() << callxargs_count << " PUSHINT -1 PUSHINT CALLXVARARGS"); + } + } else { o << "EXECUTE"; } } diff --git a/crypto/func/gen-abscode.cpp b/crypto/func/gen-abscode.cpp index 6af8e5b87..9989d10c4 100644 --- a/crypto/func/gen-abscode.cpp +++ b/crypto/func/gen-abscode.cpp @@ -92,7 +92,7 @@ bool Expr::deduce_type(const Lexem& lem) { return true; } case _VarApply: { - assert(args.size() == 2); + func_assert(args.size() == 2); TypeExpr* fun_type = TypeExpr::new_map(args[1]->e_type, TypeExpr::new_hole()); try { unify(fun_type, args[0]->e_type); @@ -107,7 +107,7 @@ bool Expr::deduce_type(const Lexem& lem) { return true; } case _Letop: { - assert(args.size() == 2); + func_assert(args.size() == 2); try { // std::cerr << "in assignment: " << args[0]->e_type << " from " << args[1]->e_type << std::endl; unify(args[0]->e_type, args[1]->e_type); @@ -122,7 +122,7 @@ bool Expr::deduce_type(const Lexem& lem) { return true; } case _LetFirst: { - assert(args.size() == 2); + func_assert(args.size() == 2); TypeExpr* rhs_type = TypeExpr::new_tensor({args[0]->e_type, TypeExpr::new_hole()}); try { // std::cerr << "in implicit assignment of a modifying method: " << rhs_type << " and " << args[1]->e_type << std::endl; @@ -140,7 +140,7 @@ bool Expr::deduce_type(const Lexem& lem) { return true; } case _CondExpr: { - assert(args.size() == 3); + func_assert(args.size() == 3); auto flag_type = TypeExpr::new_atomic(_Int); try { unify(args[0]->e_type, flag_type); @@ -204,7 +204,7 @@ int Expr::predefine_vars() { } case _Var: if (!sym) { - assert(val < 0 && here.defined()); + func_assert(val < 0 && here.defined()); if (prohibited_var_names.count(sym::symbols.get_name(~val))) { throw src::ParseError{ here, PSTRING() << "symbol `" << sym::symbols.get_name(~val) << "` cannot be redefined as a variable"}; @@ -274,7 +274,7 @@ std::vector pre_compile_tensor(const std::vector args, CodeBl arg_order.resize(args.size()); std::iota(arg_order.begin(), arg_order.end(), 0); } - assert(args.size() == arg_order.size()); + func_assert(args.size() == arg_order.size()); std::vector> res_lists(args.size()); struct ModifiedVar { @@ -310,7 +310,7 @@ std::vector pre_compile_tensor(const std::vector args, CodeBl } for (const auto& list : res_lists) { for (var_idx_t v : list) { - assert(!code.vars.at(v).on_modification.empty()); + func_assert(!code.vars.at(v).on_modification.empty()); code.vars.at(v).on_modification.pop_back(); } } @@ -339,7 +339,7 @@ std::vector Expr::pre_compile(CodeBlob& code, std::vector(sym->value); std::vector res; if (func && func->arg_order.size() == args.size() && !(code.flags & CodeBlob::_ComputeAsmLtr)) { @@ -426,7 +426,7 @@ std::vector Expr::pre_compile(CodeBlob& code, std::vectorpre_compile(code); - assert(cond.size() == 1); + func_assert(cond.size() == 1); auto rvect = new_tmp_vect(code); Op& if_op = code.emplace_back(here, Op::_If, cond); code.push_set_cur(if_op.block0); diff --git a/crypto/func/optimize.cpp b/crypto/func/optimize.cpp index f8dc3d827..74bb97ecd 100644 --- a/crypto/func/optimize.cpp +++ b/crypto/func/optimize.cpp @@ -61,9 +61,9 @@ void Optimizer::apply() { if (!p_ && !q_) { return; } - assert(p_ > 0 && p_ <= l_ && q_ >= 0 && q_ <= n && l_ <= n); + func_assert(p_ > 0 && p_ <= l_ && q_ >= 0 && q_ <= n && l_ <= n); for (int i = p_; i < l_; i++) { - assert(op_[i]); + func_assert(op_[i]); op_cons_[i]->car = std::move(op_[i]); op_cons_[i] = nullptr; } @@ -71,7 +71,7 @@ void Optimizer::apply() { code_ = std::move(code_->cdr); } for (int j = q_ - 1; j >= 0; j--) { - assert(oq_[j]); + func_assert(oq_[j]); oq_[j]->indent = indent_; code_ = AsmOpCons::cons(std::move(oq_[j]), std::move(code_)); } @@ -246,7 +246,7 @@ bool Optimizer::rewrite_const_push_xchgs() { } show_left(); auto c_op = std::move(op_[0]); - assert(c_op->is_gconst()); + func_assert(c_op->is_gconst()); StackTransform t; q_ = 0; int pos = 0; @@ -265,31 +265,31 @@ bool Optimizer::rewrite_const_push_xchgs() { if (b > pos) { oq_[q_]->b = b - 1; } - assert(apply_op(t, *oq_[q_])); + func_assert(apply_op(t, *oq_[q_])); ++q_; } } else { - assert(op_[i]->is_push(&a)); - assert(a != pos); + func_assert(op_[i]->is_push(&a)); + func_assert(a != pos); oq_[q_] = std::move(op_[i]); if (a > pos) { oq_[q_]->a = a - 1; } - assert(apply_op(t, *oq_[q_])); + func_assert(apply_op(t, *oq_[q_])); ++q_; ++pos; } } - assert(!pos); + func_assert(!pos); t.apply_push_newconst(); - assert(t <= tr_[p_ - 1]); + func_assert(t <= tr_[p_ - 1]); oq_[q_++] = std::move(c_op); show_right(); return true; } bool Optimizer::rewrite(int p, AsmOp&& new_op) { - assert(p > 0 && p <= l_); + func_assert(p > 0 && p <= l_); p_ = p; q_ = 1; show_left(); @@ -300,7 +300,7 @@ bool Optimizer::rewrite(int p, AsmOp&& new_op) { } bool Optimizer::rewrite(int p, AsmOp&& new_op1, AsmOp&& new_op2) { - assert(p > 1 && p <= l_); + func_assert(p > 1 && p <= l_); p_ = p; q_ = 2; show_left(); @@ -313,7 +313,7 @@ bool Optimizer::rewrite(int p, AsmOp&& new_op1, AsmOp&& new_op2) { } bool Optimizer::rewrite(int p, AsmOp&& new_op1, AsmOp&& new_op2, AsmOp&& new_op3) { - assert(p > 2 && p <= l_); + func_assert(p > 2 && p <= l_); p_ = p; q_ = 3; show_left(); @@ -328,7 +328,7 @@ bool Optimizer::rewrite(int p, AsmOp&& new_op1, AsmOp&& new_op2, AsmOp&& new_op3 } bool Optimizer::rewrite_nop() { - assert(p_ > 0 && p_ <= l_); + func_assert(p_ > 0 && p_ <= l_); q_ = 0; show_left(); show_right(); diff --git a/crypto/func/parse-func.cpp b/crypto/func/parse-func.cpp index 06c695bfc..15794c425 100644 --- a/crypto/func/parse-func.cpp +++ b/crypto/func/parse-func.cpp @@ -399,7 +399,7 @@ bool check_global_func(const Lexem& cur, sym_idx_t func_name = 0) { cur.loc.show_error(std::string{"undefined function `"} + symbols.get_name(func_name) + "`, defining a global function of unknown type"); def = sym::define_global_symbol(func_name, 0, cur.loc); - assert(def && "cannot define global function"); + func_assert(def && "cannot define global function"); ++undef_func_cnt; make_new_glob_func(def, TypeExpr::new_func()); // was: ... ::new_func() return true; @@ -1111,6 +1111,7 @@ blk_fl::val parse_do_stmt(Lexer& lex, CodeBlob& code) { } blk_fl::val parse_try_catch_stmt(Lexer& lex, CodeBlob& code) { + code.require_callxargs = true; lex.expect(_Try); Op& try_catch_op = code.emplace_back(lex.cur().loc, Op::_TryCatch); code.push_set_cur(try_catch_op.block0); @@ -1132,7 +1133,7 @@ blk_fl::val parse_try_catch_stmt(Lexer& lex, CodeBlob& code) { expr->predefine_vars(); expr->define_new_vars(code); try_catch_op.left = expr->pre_compile(code); - assert(try_catch_op.left.size() == 2); + func_assert(try_catch_op.left.size() == 2 || try_catch_op.left.size() == 1); blk_fl::val res1 = parse_block_stmt(lex, code); sym::close_scope(lex); code.close_pop_cur(lex.cur().loc); @@ -1295,7 +1296,7 @@ SymValAsmFunc* parse_asm_func_body(Lexer& lex, TypeExpr* func_type, const Formal } lex.next(); } - assert(arg_order.size() == (unsigned)tot_width); + func_assert(arg_order.size() == (unsigned)tot_width); } if (lex.tp() == _Mapsto) { lex.expect(_Mapsto); @@ -1487,7 +1488,7 @@ void parse_func_def(Lexer& lex) { std::cerr << "function " << func_name.str << " : " << func_type << std::endl; } SymDef* func_sym = sym::define_global_symbol(func_name.val, 0, loc); - assert(func_sym); + func_assert(func_sym); SymValFunc* func_sym_val = dynamic_cast(func_sym->value); if (func_sym->value) { if (func_sym->value->type != SymVal::_Func || !func_sym_val) { diff --git a/crypto/func/stack-transform.cpp b/crypto/func/stack-transform.cpp index 4d9b6a5fc..b52906086 100644 --- a/crypto/func/stack-transform.cpp +++ b/crypto/func/stack-transform.cpp @@ -183,7 +183,7 @@ bool StackTransform::is_permutation() const { if (!is_valid() || d) { return false; } - assert(n <= max_n); + func_assert(n <= max_n); std::array X, Y; for (int i = 0; i < n; i++) { X[i] = A[i].first; diff --git a/crypto/func/unify-types.cpp b/crypto/func/unify-types.cpp index 517299e90..f9b639c0e 100644 --- a/crypto/func/unify-types.cpp +++ b/crypto/func/unify-types.cpp @@ -132,7 +132,7 @@ void TypeExpr::replace_with(TypeExpr* te2) { } bool TypeExpr::remove_indirect(TypeExpr*& te, TypeExpr* forbidden) { - assert(te); + func_assert(te); while (te->constr == te_Indirect) { te = te->args[0]; } @@ -146,12 +146,9 @@ bool TypeExpr::remove_indirect(TypeExpr*& te, TypeExpr* forbidden) { return res; } -bool TypeExpr::remove_forall(TypeExpr*& te) { - assert(te); - if (te->constr != te_ForAll) { - return false; - } - assert(te->args.size() >= 1); +std::vector TypeExpr::remove_forall(TypeExpr*& te) { + func_assert(te && te->constr == te_ForAll); + func_assert(te->args.size() >= 1); std::vector new_vars; for (std::size_t i = 1; i < te->args.size(); i++) { new_vars.push_back(new_hole(1)); @@ -161,12 +158,12 @@ bool TypeExpr::remove_forall(TypeExpr*& te) { te = te->args[0]; remove_forall_in(te, te2, new_vars); // std::cerr << "-> " << te << std::endl; - return true; + return new_vars; } bool TypeExpr::remove_forall_in(TypeExpr*& te, TypeExpr* te2, const std::vector& new_vars) { - assert(te); - assert(te2 && te2->constr == te_ForAll); + func_assert(te); + func_assert(te2 && te2->constr == te_ForAll); if (te->constr == te_Var) { for (std::size_t i = 0; i < new_vars.size(); i++) { if (te == te2->args[i + 1]) { @@ -280,7 +277,7 @@ std::ostream& TypeExpr::print(std::ostream& os, int lex_level) { return os << "]"; } case te_Map: { - assert(args.size() == 2); + func_assert(args.size() == 2); if (lex_level > 0) { os << "("; } @@ -293,7 +290,7 @@ std::ostream& TypeExpr::print(std::ostream& os, int lex_level) { return os; } case te_ForAll: { - assert(args.size() >= 1); + func_assert(args.size() >= 1); if (lex_level > 0) { os << '('; } @@ -346,11 +343,11 @@ void check_update_widths(TypeExpr* te1, TypeExpr* te2) { check_width_compat(te1, te2); te1->minw = te2->minw = std::max(te1->minw, te2->minw); te1->maxw = te2->maxw = std::min(te1->maxw, te2->maxw); - assert(te1->minw <= te1->maxw); + func_assert(te1->minw <= te1->maxw); } void unify(TypeExpr*& te1, TypeExpr*& te2) { - assert(te1 && te2); + func_assert(te1 && te2); // std::cerr << "unify( " << te1 << " , " << te2 << " )\n"; while (te1->constr == TypeExpr::te_Indirect) { te1 = te1->args[0]; @@ -363,23 +360,37 @@ void unify(TypeExpr*& te1, TypeExpr*& te2) { } if (te1->constr == TypeExpr::te_ForAll) { TypeExpr* te = te1; - if (!TypeExpr::remove_forall(te)) { - throw UnifyError{te1, te2, "cannot remove universal type quantifier while performing type unification"}; + std::vector new_vars = TypeExpr::remove_forall(te); + for (TypeExpr* t : new_vars) { + t->was_forall_var = true; } unify(te, te2); + for (TypeExpr* t : new_vars) { + t->was_forall_var = false; + } return; } if (te2->constr == TypeExpr::te_ForAll) { TypeExpr* te = te2; - if (!TypeExpr::remove_forall(te)) { - throw UnifyError{te2, te1, "cannot remove universal type quantifier while performing type unification"}; + std::vector new_vars = TypeExpr::remove_forall(te); + for (TypeExpr* t : new_vars) { + t->was_forall_var = true; } unify(te1, te); + for (TypeExpr* t : new_vars) { + t->was_forall_var = false; + } return; } + if (te1->was_forall_var && te2->constr == TypeExpr::te_Tensor) { + throw UnifyError{te1, te2, "cannot unify generic type and tensor"}; + } + if (te2->was_forall_var && te1->constr == TypeExpr::te_Tensor) { + throw UnifyError{te2, te1, "cannot unify generic type and tensor"}; + } if (te1->constr == TypeExpr::te_Unknown) { if (te2->constr == TypeExpr::te_Unknown) { - assert(te1->value != te2->value); + func_assert(te1->value != te2->value); } if (!TypeExpr::remove_indirect(te2, te1)) { throw UnifyError{te1, te2, "type unification results in an infinite cyclic type"}; diff --git a/crypto/funcfiftlib/funcfiftlib.cpp b/crypto/funcfiftlib/funcfiftlib.cpp index 070c3e0d2..c8bf4fc56 100644 --- a/crypto/funcfiftlib/funcfiftlib.cpp +++ b/crypto/funcfiftlib/funcfiftlib.cpp @@ -92,6 +92,7 @@ td::Result compile_internal(char *config_json) { result_obj("status", "ok"); result_obj("codeBoc", td::base64_encode(boc)); result_obj("fiftCode", escape_json(outs.str())); + result_obj("codeHashHex", code_cell->get_hash().to_hex()); result_obj.leave(); outs.clear(); diff --git a/doc/catchain.tex b/doc/catchain.tex index b3cae642b..b1a27164d 100644 --- a/doc/catchain.tex +++ b/doc/catchain.tex @@ -243,7 +243,7 @@ Once a fork (created by~$i$) is detected (by another process~$j$), i.e.\ $j$ learns about two different messages $m_{i,s}$ and $m'_{i,s}$ created by $i$ and having same height $s$ (usually this happens while recursively downloading dependencies of some other messages), $j$ starts ignoring~$i$ and all of its subsequent messages. They are not accepted and not broadcast further. However, messages created by~$i$ prior to the fork detection may be still downloaded if they are referred to in messages (blocks) created by processes that did not see this fork before referring to such messages created by~$i$. \nxsubpoint\emb{Accepting messages from a ``bad'' process is bad}\label{sp:no.bad.accept} -Furthermore, if process $i$ learns about a fork created by process $j$, then $i$ shows this to its neighbors by creating a new service broadcast message that contains the corresponding fork proof (cf.~\ptref{sp:fork.proofs}). Afterwards, this and all subsequent messages of $j$ cannot directly depend on any messages by the known ``bad'' producer $i$ (but they still can refer to messages from another party $k$ that directly or indirectly refer to messages of~$i$ if no fork by~$i$ was known to $k$ at the time when the referring message was created). If $j$ violates this restriction and creates messages with such invalid references, these messages will be discarded by all honest processes in the group. +Furthermore, if process $i$ learns about a fork created by process $j$, then $i$ shows this to its neighbors by creating a new service broadcast message that contains the corresponding fork proof (cf.~\ptref{sp:fork.proofs}). Afterwards, this and all subsequent messages of $i$ cannot directly depend on any messages by the known ``bad'' producer $j$ (but they still can refer to messages from another party $k$ that directly or indirectly refer to messages of~$i$ if no fork by~$i$ was known to $k$ at the time when the referring message was created). If $i$ violates this restriction and creates messages with such invalid references, these messages will be discarded by all honest processes in the group. \nxsubpoint\emb{The set of ``bad'' group members is a part of the intrinsic state}\label{sp:bad.proc.set} Each process~$i$ keeps its own copy of the set of known ``bad'' processes in the group, i.e., those processes that have created at least one fork or have violated \ptref{sp:no.bad.accept}. This set is updated by adding~$j$ into it as soon as $i$ learns about a fork created by~$j$ (or about a violation of~\ptref{sp:no.bad.accept} by $j$); after that, a callback provided by the higher-level protocol is invoked. This set is used when a new broadcast message arrives: if the sender is bad, then the message is ignored and discarded. diff --git a/emulator/CMakeLists.txt b/emulator/CMakeLists.txt index 4e67d32cc..1389050c6 100644 --- a/emulator/CMakeLists.txt +++ b/emulator/CMakeLists.txt @@ -1,5 +1,7 @@ cmake_minimum_required(VERSION 3.0.2 FATAL_ERROR) +option(BUILD_SHARED_LIBS "Use \"OFF\" for a static build." ON) + if (NOT OPENSSL_FOUND) find_package(OpenSSL REQUIRED) endif() @@ -27,7 +29,12 @@ include(GenerateExportHeader) add_library(emulator_static STATIC ${EMULATOR_STATIC_SOURCE}) target_link_libraries(emulator_static PUBLIC ton_crypto ton_block smc-envelope) -add_library(emulator STATIC ${EMULATOR_SOURCE} ${EMULATOR_HEADERS}) +if (NOT USE_EMSCRIPTEN AND BUILD_SHARED_LIBS) + add_library(emulator SHARED ${EMULATOR_SOURCE} ${EMULATOR_HEADERS}) +else() + add_library(emulator STATIC ${EMULATOR_SOURCE} ${EMULATOR_HEADERS}) +endif() + target_link_libraries(emulator PUBLIC emulator_static) generate_export_header(emulator EXPORT_FILE_NAME ${CMAKE_CURRENT_BINARY_DIR}/emulator_export.h) target_include_directories(emulator PUBLIC diff --git a/flake.nix b/flake.nix index e7b0422e0..d3fca85c3 100644 --- a/flake.nix +++ b/flake.nix @@ -50,6 +50,9 @@ "-static-libstdc++" ]); + GIT_REVISION = if self ? rev then self.rev else "dirty"; + GIT_REVISION_DATE = (builtins.concatStringsSep "-" (builtins.match "(.{4})(.{2})(.{2}).*" self.lastModifiedDate)) + " " + (builtins.concatStringsSep ":" (builtins.match "^........(.{2})(.{2})(.{2}).*" self.lastModifiedDate)); + postInstall = '' moveToOutput bin "$bin" ''; diff --git a/git_watcher.cmake b/git_watcher.cmake index ab135e808..78e57ba14 100644 --- a/git_watcher.cmake +++ b/git_watcher.cmake @@ -139,6 +139,8 @@ function(GetGitState _working_dir) RunGitCommand(show -s "--format=%H" ${object}) if(exit_code EQUAL 0) set(ENV{GIT_HEAD_SHA1} ${output}) + else() + set(ENV{GIT_HEAD_SHA1} "$ENV{GIT_REVISION}") endif() RunGitCommand(show -s "--format=%an" ${object}) @@ -154,6 +156,8 @@ function(GetGitState _working_dir) RunGitCommand(show -s "--format=%ci" ${object}) if(exit_code EQUAL 0) set(ENV{GIT_COMMIT_DATE_ISO8601} "${output}") + else() + set(ENV{GIT_COMMIT_DATE_ISO8601} "$ENV{GIT_REVISION_DATE}") endif() RunGitCommand(show -s "--format=%s" ${object}) diff --git a/recent_changelog.md b/recent_changelog.md new file mode 100644 index 000000000..39ca5c942 --- /dev/null +++ b/recent_changelog.md @@ -0,0 +1,8 @@ +## 2023.05 Update +1. Archive manager optimization +2. A series of catchain (basic consensus protocol) security improvements +3. Update for Fift libraries and FunC: better error-handling, fixes for `catch` stack recovery +4. A series of out message queue handling optimization (already deployed during emergency upgrades between releases) +5. Improvement of binaries portability + +Besides the work of the core team, this update is based on the efforts of @aleksej-paschenko (portability improvement), [Disintar team](https://github.com/disintar/) (archive manager optimization) and [sec3-service](https://github.com/sec3-service) security auditors (funC improvements). diff --git a/tdutils/td/utils/Timer.cpp b/tdutils/td/utils/Timer.cpp index f33a30dc8..0f6a7d6ed 100644 --- a/tdutils/td/utils/Timer.cpp +++ b/tdutils/td/utils/Timer.cpp @@ -78,9 +78,12 @@ void PerfWarningTimer::reset() { return; } double duration = Time::now() - start_at_; - //LOG_IF(WARNING, duration > max_duration_) - //<< "SLOW: " << tag("name", name_) << tag("duration", format::as_time(duration)); - callback_(duration); + if (callback_) { + callback_(duration); + } else { + LOG_IF(WARNING, duration > max_duration_) + << "SLOW: " << tag("name", name_) << tag("duration", format::as_time(duration)); + } start_at_ = 0; } diff --git a/tdutils/td/utils/Timer.h b/tdutils/td/utils/Timer.h index 64f3e934d..18ea35d7c 100644 --- a/tdutils/td/utils/Timer.h +++ b/tdutils/td/utils/Timer.h @@ -46,7 +46,7 @@ class Timer { class PerfWarningTimer { public: - explicit PerfWarningTimer(string name, double max_duration = 0.1, std::function&& callback = [] (double) {}); + explicit PerfWarningTimer(string name, double max_duration = 0.1, std::function&& callback = {}); PerfWarningTimer(const PerfWarningTimer &) = delete; PerfWarningTimer &operator=(const PerfWarningTimer &) = delete; PerfWarningTimer(PerfWarningTimer &&other); diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index 66e6f4a69..346d91522 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -277,14 +277,10 @@ catchain.differenceFork left:catchain.block.dep right:catchain.block.dep = catch catchain.blockNotFound = catchain.BlockResult; catchain.blockResult block:catchain.block = catchain.BlockResult; -catchain.sent cnt:int = catchain.Sent; - ---functions--- catchain.getBlock block:int256 = catchain.BlockResult; -catchain.getBlocks blocks:(vector int256) = catchain.Sent; catchain.getDifference rt:(vector int) = catchain.Difference; -catchain.getBlockHistory block:int256 height:long stop_if:(vector int256) = catchain.Sent; //catchain.getForkDifference src:int fork:catchain.fork = catchain.ForkDifference; ---types--- diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index 8ce7a5791..5b8e1c725 100644 Binary files a/tl/generate/scheme/ton_api.tlo and b/tl/generate/scheme/ton_api.tlo differ diff --git a/validator-engine/validator-engine.cpp b/validator-engine/validator-engine.cpp index 02ab42f9e..65b93df71 100644 --- a/validator-engine/validator-engine.cpp +++ b/validator-engine/validator-engine.cpp @@ -1319,9 +1319,6 @@ td::Status ValidatorEngine::load_global_config() { } CHECK(mode == ton::validator::ValidatorManagerOptions::ShardCheckMode::m_validate); return true; - /*ton::ShardIdFull p{ton::basechainId, ((cc_seqno * 1ull % 4) << 62) + 1}; - auto s = ton::shard_prefix(p, 2); - return shard.is_masterchain() || ton::shard_intersects(shard, s);*/ }); if (state_ttl_ != 0) { validator_options_.write().set_state_ttl(state_ttl_); @@ -3647,34 +3644,61 @@ int main(int argc, char *argv[]) { logger_ = td::TsFileLog::create(fname.str()).move_as_ok(); td::log_interface = logger_.get(); }); - p.add_option('s', "state-ttl", "state will be gc'd after this time (in seconds) default=3600", [&](td::Slice fname) { - auto v = td::to_double(fname); - acts.push_back([&x, v]() { td::actor::send_closure(x, &ValidatorEngine::set_state_ttl, v); }); - }); - p.add_option('m', "mempool-num", "Maximal number of mempool external message", [&](td::Slice fname) { + p.add_checked_option('s', "state-ttl", "state will be gc'd after this time (in seconds) default=3600", + [&](td::Slice fname) { + auto v = td::to_double(fname); + if (v <= 0) { + return td::Status::Error("state-ttl should be positive"); + } + acts.push_back([&x, v]() { td::actor::send_closure(x, &ValidatorEngine::set_state_ttl, v); }); + return td::Status::OK(); + }); + p.add_checked_option('m', "mempool-num", "Maximal number of mempool external message", [&](td::Slice fname) { auto v = td::to_double(fname); + if (v < 0) { + return td::Status::Error("mempool-num should be non-negative"); + } acts.push_back([&x, v]() { td::actor::send_closure(x, &ValidatorEngine::set_max_mempool_num, v); }); + return td::Status::OK(); }); - p.add_option('b', "block-ttl", "blocks will be gc'd after this time (in seconds) default=7*86400", - [&](td::Slice fname) { - auto v = td::to_double(fname); - acts.push_back([&x, v]() { td::actor::send_closure(x, &ValidatorEngine::set_block_ttl, v); }); - }); - p.add_option('A', "archive-ttl", "archived blocks will be deleted after this time (in seconds) default=365*86400", - [&](td::Slice fname) { - auto v = td::to_double(fname); - acts.push_back([&x, v]() { td::actor::send_closure(x, &ValidatorEngine::set_archive_ttl, v); }); - }); - p.add_option('K', "key-proof-ttl", "key blocks will be deleted after this time (in seconds) default=365*86400*10", - [&](td::Slice fname) { - auto v = td::to_double(fname); - acts.push_back([&x, v]() { td::actor::send_closure(x, &ValidatorEngine::set_key_proof_ttl, v); }); - }); - p.add_option('S', "sync-before", "in initial sync download all blocks for last given seconds default=3600", - [&](td::Slice fname) { - auto v = td::to_double(fname); - acts.push_back([&x, v]() { td::actor::send_closure(x, &ValidatorEngine::set_sync_ttl, v); }); - }); + p.add_checked_option('b', "block-ttl", "blocks will be gc'd after this time (in seconds) default=7*86400", + [&](td::Slice fname) { + auto v = td::to_double(fname); + if (v <= 0) { + return td::Status::Error("block-ttl should be positive"); + } + acts.push_back([&x, v]() { td::actor::send_closure(x, &ValidatorEngine::set_block_ttl, v); }); + return td::Status::OK(); + }); + p.add_checked_option( + 'A', "archive-ttl", "archived blocks will be deleted after this time (in seconds) default=365*86400", + [&](td::Slice fname) { + auto v = td::to_double(fname); + if (v <= 0) { + return td::Status::Error("archive-ttl should be positive"); + } + acts.push_back([&x, v]() { td::actor::send_closure(x, &ValidatorEngine::set_archive_ttl, v); }); + return td::Status::OK(); + }); + p.add_checked_option( + 'K', "key-proof-ttl", "key blocks will be deleted after this time (in seconds) default=365*86400*10", + [&](td::Slice fname) { + auto v = td::to_double(fname); + if (v <= 0) { + return td::Status::Error("key-proof-ttl should be positive"); + } + acts.push_back([&x, v]() { td::actor::send_closure(x, &ValidatorEngine::set_key_proof_ttl, v); }); + return td::Status::OK(); + }); + p.add_checked_option('S', "sync-before", "in initial sync download all blocks for last given seconds default=3600", + [&](td::Slice fname) { + auto v = td::to_double(fname); + if (v <= 0) { + return td::Status::Error("sync-before should be positive"); + } + acts.push_back([&x, v]() { td::actor::send_closure(x, &ValidatorEngine::set_sync_ttl, v); }); + return td::Status::OK(); + }); p.add_option('T', "truncate-db", "truncate db (with specified seqno as new top masterchain block seqno)", [&](td::Slice fname) { auto v = td::to_integer(fname); diff --git a/validator-session/CMakeLists.txt b/validator-session/CMakeLists.txt index dc837219c..4931e464d 100644 --- a/validator-session/CMakeLists.txt +++ b/validator-session/CMakeLists.txt @@ -9,6 +9,7 @@ set(VALIDATOR_SESSION_SOURCE validator-session-description.cpp validator-session-state.cpp validator-session.cpp + validator-session-round-attempt-state.cpp persistent-vector.h validator-session-description.h @@ -16,7 +17,7 @@ set(VALIDATOR_SESSION_SOURCE validator-session-state.h validator-session.h validator-session.hpp -) + validator-session-round-attempt-state.h) add_library(validatorsession STATIC ${VALIDATOR_SESSION_SOURCE}) diff --git a/validator-session/persistent-vector.h b/validator-session/persistent-vector.h index 77852f80a..3491164ca 100644 --- a/validator-session/persistent-vector.h +++ b/validator-session/persistent-vector.h @@ -322,7 +322,6 @@ class CntVector : public ValidatorSessionDescription::RootObject { CHECK(idx < size()); return data_[idx]; } - //const T& at(size_t idx) const; private: const td::uint32 data_size_; @@ -546,7 +545,6 @@ class CntVector : public ValidatorSessionDescription::RootObject { CHECK(idx < max_size()); return get_bit(data_, idx); } - //const T& at(size_t idx) const; private: const td::uint32 data_size_; @@ -734,10 +732,6 @@ class CntSortedVector : public ValidatorSessionDescription::RootObject { return CntSortedVector::create(desc, std::move(v)); } - /*static const CntSortedVector* merge(ValidatorSessionDescription& desc, const CntSortedVector* l, - const CntSortedVector* r) { - return merge(desc, l, r, [](T l, T r) { return l; }); - }*/ static const CntSortedVector* push(ValidatorSessionDescription& desc, const CntSortedVector* v, T value) { if (!v) { return create(desc, std::vector{value}); @@ -795,7 +789,6 @@ class CntSortedVector : public ValidatorSessionDescription::RootObject { CHECK(idx < size()); return data_[idx]; } - //const T& at(size_t idx) const; private: const td::uint32 data_size_; diff --git a/validator-session/validator-session-description.cpp b/validator-session/validator-session-description.cpp index eb4c3a8ea..34dc90212 100644 --- a/validator-session/validator-session-description.cpp +++ b/validator-session/validator-session-description.cpp @@ -51,13 +51,6 @@ ValidatorSessionDescriptionImpl::ValidatorSessionDescriptionImpl(ValidatorSessio CHECK(it != rev_sources_.end()); self_idx_ = it->second; - pdata_temp_ptr_ = 0; - pdata_temp_size_ = 1 << 27; - pdata_temp_ = new td::uint8[pdata_temp_size_]; - - pdata_perm_size_ = 1ull << 27; - pdata_perm_ptr_ = 0; - for (auto &el : cache_) { Cached v{nullptr}; el.store(v, std::memory_order_relaxed); @@ -162,50 +155,70 @@ void ValidatorSessionDescriptionImpl::update_hash(const RootObject *obj, HashTyp } void *ValidatorSessionDescriptionImpl::alloc(size_t size, size_t align, bool temp) { - CHECK(align && !(align & (align - 1))); // align should be a power of 2 + return (temp ? mem_temp_ : mem_perm_).alloc(size, align); +} + +bool ValidatorSessionDescriptionImpl::is_persistent(const void *ptr) const { + return mem_perm_.contains(ptr); +} + +std::unique_ptr ValidatorSessionDescription::create( + ValidatorSessionOptions opts, std::vector &nodes, PublicKeyHash local_id) { + return std::make_unique(std::move(opts), nodes, local_id); +} + +ValidatorSessionDescriptionImpl::MemPool::MemPool(size_t chunk_size) : chunk_size_(chunk_size) { +} + +ValidatorSessionDescriptionImpl::MemPool::~MemPool() { + for (auto &v : data_) { + delete[] v; + } +} + +void *ValidatorSessionDescriptionImpl::MemPool::alloc(size_t size, size_t align) { + CHECK(align && !(align & (align - 1))); // align should be a power of 2 + CHECK(size + align <= chunk_size_); auto get_padding = [&](const uint8_t* ptr) { return (-(size_t)ptr) & (align - 1); }; - if (temp) { - pdata_temp_ptr_ += get_padding(pdata_temp_ + pdata_temp_ptr_); - auto s = pdata_temp_ptr_; - pdata_temp_ptr_ += size; - CHECK(s + size <= pdata_temp_size_); - return static_cast(pdata_temp_ + s); - } else { - while (true) { - size_t idx = pdata_perm_ptr_ / pdata_perm_size_; - if (idx < pdata_perm_.size()) { - auto ptr = pdata_perm_[idx] + (pdata_perm_ptr_ % pdata_perm_size_); - pdata_perm_ptr_ += get_padding(ptr); - ptr += get_padding(ptr); - pdata_perm_ptr_ += size; - if (pdata_perm_ptr_ <= pdata_perm_.size() * pdata_perm_size_) { - return static_cast(ptr); - } + while (true) { + size_t idx = ptr_ / chunk_size_; + if (idx < data_.size()) { + auto ptr = data_[idx] + (ptr_ % chunk_size_); + ptr_ += get_padding(ptr); + ptr += get_padding(ptr); + ptr_ += size; + if (ptr_ <= data_.size() * chunk_size_) { + return static_cast(ptr); + } else { + ptr_ = data_.size() * chunk_size_; } - pdata_perm_.push_back(new td::uint8[pdata_perm_size_]); } + data_.push_back(new td::uint8[chunk_size_]); } } -bool ValidatorSessionDescriptionImpl::is_persistent(const void *ptr) const { +void ValidatorSessionDescriptionImpl::MemPool::clear() { + while (data_.size() > 1) { + delete[] data_.back(); + data_.pop_back(); + } + ptr_ = 0; +} + +bool ValidatorSessionDescriptionImpl::MemPool::contains(const void* ptr) const { if (ptr == nullptr) { return true; } - for (auto &v : pdata_perm_) { - if (ptr >= v && ptr <= v + pdata_perm_size_) { + for (auto &v : data_) { + if (ptr >= v && ptr <= v + chunk_size_) { return true; } } return false; } -std::unique_ptr ValidatorSessionDescription::create( - ValidatorSessionOptions opts, std::vector &nodes, PublicKeyHash local_id) { - return std::make_unique(std::move(opts), nodes, local_id); -} - } // namespace validatorsession } // namespace ton diff --git a/validator-session/validator-session-description.hpp b/validator-session/validator-session-description.hpp index d65369ace..0c662b7a2 100644 --- a/validator-session/validator-session-description.hpp +++ b/validator-session/validator-session-description.hpp @@ -50,20 +50,30 @@ class ValidatorSessionDescriptionImpl : public ValidatorSessionDescription { td::uint32 self_idx_; static constexpr td::uint32 cache_size = (1 << 20); + static constexpr size_t mem_chunk_size_perm = (1 << 27); + static constexpr size_t mem_chunk_size_temp = (1 << 27); struct Cached { const RootObject *ptr; }; std::array, cache_size> cache_; - //std::array, cache_size> temp_cache_; - td::uint8 *pdata_temp_; - size_t pdata_temp_ptr_; - size_t pdata_temp_size_; + class MemPool { + public: + explicit MemPool(size_t chunk_size); + ~MemPool(); + void *alloc(size_t size, size_t align); + void clear(); + bool contains(const void* ptr) const; + + private: + size_t chunk_size_; + std::vector data_; + size_t ptr_ = 0; + }; + MemPool mem_perm_ = MemPool(mem_chunk_size_perm); + MemPool mem_temp_ = MemPool(mem_chunk_size_temp); - size_t pdata_perm_size_; - std::vector pdata_perm_; - size_t pdata_perm_ptr_; std::atomic reuse_{0}; public: @@ -116,7 +126,7 @@ class ValidatorSessionDescriptionImpl : public ValidatorSessionDescription { void update_hash(const RootObject *obj, HashType hash) override; void *alloc(size_t size, size_t align, bool temp) override; void clear_temp_memory() override { - pdata_temp_ptr_ = 0; + mem_temp_.clear(); } bool is_persistent(const void *ptr) const override; HashType compute_hash(td::Slice data) const override; @@ -152,12 +162,6 @@ class ValidatorSessionDescriptionImpl : public ValidatorSessionDescription { const ValidatorSessionOptions &opts() const override { return opts_; } - ~ValidatorSessionDescriptionImpl() { - delete[] pdata_temp_; - for (auto &x : pdata_perm_) { - delete[] x; - } - } }; } // namespace validatorsession diff --git a/validator-session/validator-session-round-attempt-state.cpp b/validator-session/validator-session-round-attempt-state.cpp new file mode 100644 index 000000000..33c3d8047 --- /dev/null +++ b/validator-session/validator-session-round-attempt-state.cpp @@ -0,0 +1,486 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . + + Copyright 2017-2020 Telegram Systems LLP +*/ +#include "validator-session-state.h" +#include "td/utils/Random.h" +#include "auto/tl/ton_api.hpp" + +#include + +namespace ton { + +namespace validatorsession { + +static const SessionVoteCandidate* get_candidate(const VoteVector* vec, ValidatorSessionCandidateId id) { + if (!vec) { + return nullptr; + } + auto size = vec->size(); + auto v = vec->data(); + for (td::uint32 i = 0; i < size; i++) { + if (v[i]->get_id() == id) { + return v[i]; + } + } + return nullptr; +} + +// +// +// SessionBlockCandidateSignature +// +// + +const SessionBlockCandidateSignature* SessionBlockCandidateSignature::merge(ValidatorSessionDescription& desc, + const SessionBlockCandidateSignature* l, + const SessionBlockCandidateSignature* r) { + if (!l) { + return r; + } + if (!r) { + return l; + } + if (l == r) { + return l; + } + if (l->as_slice() < r->as_slice()) { + return l; + } else { + return r; + } +} + +// +// +// SessionBlockCandidate +// +// + +bool SessionBlockCandidate::check_block_is_approved(ValidatorSessionDescription& desc) const { + ValidatorWeight w = 0; + for (td::uint32 i = 0; i < desc.get_total_nodes(); i++) { + if (approved_by_->at(i)) { + w += desc.get_node_weight(i); + if (w >= desc.get_cutoff_weight()) { + return true; + } + } + } + return false; +} + +const SessionBlockCandidate* SessionBlockCandidate::merge(ValidatorSessionDescription& desc, + const SessionBlockCandidate* l, + const SessionBlockCandidate* r) { + if (!l) { + return r; + } + if (!r) { + return l; + } + if (l == r) { + return l; + } + CHECK(l->get_id() == r->get_id()); + auto v = SessionBlockCandidateSignatureVector::merge( + desc, l->approved_by_, r->approved_by_, + [&](const SessionBlockCandidateSignature* l, const SessionBlockCandidateSignature* r) { + return SessionBlockCandidateSignature::merge(desc, l, r); + }); + return SessionBlockCandidate::create(desc, l->block_, std::move(v)); +} + +// +// +// SessionVoteCandidate +// +// + +bool SessionVoteCandidate::check_block_is_voted(ValidatorSessionDescription& desc) const { + ValidatorWeight w = 0; + for (td::uint32 i = 0; i < desc.get_total_nodes(); i++) { + if (voted_by_->at(i)) { + w += desc.get_node_weight(i); + if (w >= desc.get_cutoff_weight()) { + return true; + } + } + } + return false; +} + +const SessionVoteCandidate* SessionVoteCandidate::merge(ValidatorSessionDescription& desc, + const SessionVoteCandidate* l, const SessionVoteCandidate* r) { + if (!l) { + return r; + } + if (!r) { + return l; + } + if (l == r) { + return l; + } + CHECK(l->get_id() == r->get_id()); + auto v = CntVector::merge(desc, l->voted_by_, r->voted_by_); + return SessionVoteCandidate::create(desc, l->block_, std::move(v)); +} + +// +// +// ATTEMPT STATE +// +// + +const ValidatorSessionRoundAttemptState* ValidatorSessionRoundAttemptState::merge( + ValidatorSessionDescription& desc, const ValidatorSessionRoundAttemptState* left, + const ValidatorSessionRoundAttemptState* right) { + if (!left) { + return right; + } + if (!right) { + return left; + } + if (left == right) { + return left; + } + CHECK(left->seqno_ == right->seqno_); + + const SentBlock* vote_for = nullptr; + bool vote_for_inited = false; + if (!left->vote_for_inited_) { + vote_for = right->vote_for_; + vote_for_inited = right->vote_for_inited_; + } else if (!right->vote_for_inited_) { + vote_for = left->vote_for_; + vote_for_inited = left->vote_for_inited_; + } else if (left->vote_for_ == right->vote_for_) { + vote_for_inited = true; + vote_for = left->vote_for_; + } else { + auto l = SentBlock::get_block_id(left->vote_for_); + auto r = SentBlock::get_block_id(right->vote_for_); + + vote_for_inited = true; + if (l < r) { + vote_for = left->vote_for_; + } else { + vote_for = right->vote_for_; + } + } + + auto precommitted = CntVector::merge(desc, left->precommitted_, right->precommitted_); + auto votes = VoteVector::merge(desc, left->votes_, right->votes_, + [&](const SessionVoteCandidate* l, const SessionVoteCandidate* r) { + return SessionVoteCandidate::merge(desc, l, r); + }); + + return ValidatorSessionRoundAttemptState::create(desc, left->seqno_, std::move(votes), std::move(precommitted), + vote_for, vote_for_inited); +} + +const ValidatorSessionRoundAttemptState* ValidatorSessionRoundAttemptState::action( + ValidatorSessionDescription& desc, const ValidatorSessionRoundAttemptState* state, td::uint32 src_idx, + td::uint32 att, const ton_api::validatorSession_message_voteFor& act, const ValidatorSessionRoundState* round) { + if (state->vote_for_inited_) { + VLOG(VALIDATOR_SESSION_WARNING) << "[validator session][node " << desc.get_source_id(src_idx) << "][" << act + << "]: invalid message: duplicate VOTEFOR"; + return state; + } + if (src_idx != desc.get_vote_for_author(att)) { + VLOG(VALIDATOR_SESSION_WARNING) << "[validator session][node " << desc.get_source_id(src_idx) << "][" << act + << "]: invalid message: bad VOTEFOR author"; + return state; + } + if (round->get_first_attempt(src_idx) == 0 && desc.opts().max_round_attempts > 0) { + VLOG(VALIDATOR_SESSION_WARNING) << "[validator session][node " << desc.get_source_id(src_idx) << "][" << act + << "]: invalid message: too early for VOTEFOR"; + return state; + } + if (round->get_first_attempt(src_idx) + desc.opts().max_round_attempts > att && desc.opts().max_round_attempts > 0) { + VLOG(VALIDATOR_SESSION_WARNING) << "[validator session][node " << desc.get_source_id(src_idx) << "][" << act + << "]: invalid message: too early for VOTEFOR"; + return state; + } + + auto x = round->get_block(act.candidate_); + + if (!x) { + VLOG(VALIDATOR_SESSION_WARNING) << "[validator session][node " << desc.get_source_id(src_idx) << "][" << act + << "]: invalid message: VOTEFOR for not submitted block"; + return state; + } + if (!x->check_block_is_approved(desc)) { + VLOG(VALIDATOR_SESSION_WARNING) << "[validator session][node " << desc.get_source_id(src_idx) << "][" << act + << "]: invalid message: VOTEFOR for not approved block"; + return state; + } + + return ValidatorSessionRoundAttemptState::create(desc, state->seqno_, state->votes_, state->precommitted_, + x->get_block(), true); +} + +const ValidatorSessionRoundAttemptState* ValidatorSessionRoundAttemptState::action( + ValidatorSessionDescription& desc, const ValidatorSessionRoundAttemptState* state, td::uint32 src_idx, + td::uint32 att, const ton_api::validatorSession_message_vote& act, const ValidatorSessionRoundState* round) { + bool made; + return make_one(desc, state, src_idx, att, round, &act, made); +} + +const ValidatorSessionRoundAttemptState* ValidatorSessionRoundAttemptState::action( + ValidatorSessionDescription& desc, const ValidatorSessionRoundAttemptState* state, td::uint32 src_idx, + td::uint32 att, const ton_api::validatorSession_message_precommit& act, const ValidatorSessionRoundState* round) { + bool made; + return make_one(desc, state, src_idx, att, round, &act, made); +} + +const ValidatorSessionRoundAttemptState* ValidatorSessionRoundAttemptState::action( + ValidatorSessionDescription& desc, const ValidatorSessionRoundAttemptState* state, td::uint32 src_idx, + td::uint32 att, const ton_api::validatorSession_message_empty& act, const ValidatorSessionRoundState* round) { + bool made; + return make_one(desc, state, src_idx, att, round, &act, made); +} + +const ValidatorSessionRoundAttemptState* ValidatorSessionRoundAttemptState::try_vote( + ValidatorSessionDescription& desc, const ValidatorSessionRoundAttemptState* state, td::uint32 src_idx, + td::uint32 att, const ValidatorSessionRoundState* round, const ton_api::validatorSession_round_Message* act, + bool& made) { + made = false; + if (state->check_vote_received_from(src_idx)) { + return state; + } + auto found = false; + auto block = round->choose_block_to_vote(desc, src_idx, att, state->vote_for_, state->vote_for_inited_, found); + if (!found) { + return state; + } + auto block_id = SentBlock::get_block_id(block); + made = true; + + if (act) { + if (act->get_id() != ton_api::validatorSession_message_vote::ID) { + VLOG(VALIDATOR_SESSION_WARNING) << "[validator session][node " << desc.get_source_id(src_idx) << "][" << act + << "]: expected VOTE(" << block_id << ")"; + } else { + auto x = static_cast(act); + if (x->candidate_ != block_id) { + VLOG(VALIDATOR_SESSION_WARNING) << "[validator session][node " << desc.get_source_id(src_idx) << "][" << act + << "]: expected VOTE(" << block_id << ")"; + } + } + } else { + VLOG(VALIDATOR_SESSION_WARNING) << "[validator session][node " << desc.get_source_id(src_idx) + << "]: making implicit VOTE(" << block_id << ")"; + } + + auto candidate = get_candidate(state->votes_, block_id); + if (!candidate) { + candidate = SessionVoteCandidate::create(desc, block); + } + candidate = SessionVoteCandidate::push(desc, candidate, src_idx); + auto v = VoteVector::push(desc, state->votes_, candidate); + return ValidatorSessionRoundAttemptState::create(desc, state->seqno_, std::move(v), state->precommitted_, + state->vote_for_, state->vote_for_inited_); +} + +const ValidatorSessionRoundAttemptState* ValidatorSessionRoundAttemptState::try_precommit( + ValidatorSessionDescription& desc, const ValidatorSessionRoundAttemptState* state, td::uint32 src_idx, + td::uint32 att, const ValidatorSessionRoundState* round, const ton_api::validatorSession_round_Message* act, + bool& made) { + made = false; + if (state->check_precommit_received_from(src_idx)) { + return state; + } + bool found; + auto block = state->get_voted_block(desc, found); + if (!found) { + return state; + } + made = true; + auto block_id = SentBlock::get_block_id(block); + + if (act) { + if (act->get_id() != ton_api::validatorSession_message_precommit::ID) { + VLOG(VALIDATOR_SESSION_WARNING) << "[validator session][node " << desc.get_source_id(src_idx) << "][" << act + << "]: expected PRECOMMIT(" << block_id << ")"; + } else { + auto x = static_cast(act); + if (x->candidate_ != block_id) { + VLOG(VALIDATOR_SESSION_WARNING) << "[validator session][node " << desc.get_source_id(src_idx) << "][" << act + << "]: expected PRECOMMIT(" << block_id << ")"; + } + } + } else { + VLOG(VALIDATOR_SESSION_WARNING) << "[validator session][node " << desc.get_source_id(src_idx) + << "]: making implicit PRECOMMIT(" << block_id << ")"; + } + + auto v = CntVector::change(desc, state->precommitted_, src_idx, true); + return ValidatorSessionRoundAttemptState::create(desc, state->seqno_, state->votes_, std::move(v), state->vote_for_, + state->vote_for_inited_); +} + +const ValidatorSessionRoundAttemptState* ValidatorSessionRoundAttemptState::make_one( + ValidatorSessionDescription& desc, const ValidatorSessionRoundAttemptState* state, td::uint32 src_idx, + td::uint32 att, const ValidatorSessionRoundState* round, const ton_api::validatorSession_round_Message* act, + bool& made) { + made = false; + state = try_vote(desc, state, src_idx, att, round, act, made); + if (made) { + return state; + } + state = try_precommit(desc, state, src_idx, att, round, act, made); + if (made) { + return state; + } + if (act && act->get_id() != ton_api::validatorSession_message_empty::ID) { + VLOG(VALIDATOR_SESSION_WARNING) << "[validator session][node " << desc.get_source_id(src_idx) << "][" << act + << "]: invalid message: expected EMPTY"; + } + return state; +} + +const ValidatorSessionRoundAttemptState* ValidatorSessionRoundAttemptState::action( + ValidatorSessionDescription& desc, const ValidatorSessionRoundAttemptState* state, td::uint32 src_idx, + td::uint32 att, const ton_api::validatorSession_round_Message* act, const ValidatorSessionRoundState* round) { + ton_api::downcast_call(*const_cast(act), + [&](auto& obj) { state = action(desc, state, src_idx, att, obj, round); }); + return state; +} + +bool ValidatorSessionRoundAttemptState::check_vote_received_from(td::uint32 src_idx) const { + if (!votes_) { + return false; + } + auto size = votes_->size(); + auto v = votes_->data(); + for (td::uint32 i = 0; i < size; i++) { + if (v[i]->check_block_is_voted_by(src_idx)) { + return true; + } + } + return false; +} + +bool ValidatorSessionRoundAttemptState::check_precommit_received_from(td::uint32 src_idx) const { + return precommitted_->at(src_idx); +} + +const SentBlock* ValidatorSessionRoundAttemptState::get_voted_block(ValidatorSessionDescription& desc, bool& f) const { + f = false; + if (!votes_) { + return nullptr; + } + + auto size = votes_->size(); + auto v = votes_->data(); + for (td::uint32 i = 0; i < size; i++) { + if (v[i]->check_block_is_voted(desc)) { + f = true; + return v[i]->get_block(); + } + } + return nullptr; +} + +bool ValidatorSessionRoundAttemptState::check_attempt_is_precommitted(ValidatorSessionDescription& desc) const { + ValidatorWeight weight = 0; + for (td::uint32 i = 0; i < desc.get_total_nodes(); i++) { + if (precommitted_->at(i)) { + weight += desc.get_node_weight(i); + if (weight >= desc.get_cutoff_weight()) { + return true; + } + } + } + return false; +} + +tl_object_ptr ValidatorSessionRoundAttemptState::create_action( + ValidatorSessionDescription& desc, const ValidatorSessionRoundState* round, td::uint32 src_idx, + td::uint32 att) const { + if (!check_vote_received_from(src_idx)) { + auto found = false; + auto B = round->choose_block_to_vote(desc, src_idx, att, vote_for_, vote_for_inited_, found); + if (found) { + auto block_id = SentBlock::get_block_id(B); + return create_tl_object(round->get_seqno(), seqno_, block_id); + } + } + if (!check_precommit_received_from(src_idx)) { + bool f = false; + auto B = get_voted_block(desc, f); + + if (f) { + auto block_id = SentBlock::get_block_id(B); + + return create_tl_object(round->get_seqno(), seqno_, block_id); + } + } + + return create_tl_object(round->get_seqno(), seqno_); +} + +void ValidatorSessionRoundAttemptState::dump(ValidatorSessionDescription& desc, td::StringBuilder& sb) const { + sb << "attempt=" << seqno_ << "\n"; + sb << ">>>>\n"; + + if (vote_for_inited_) { + sb << "vote_for=" << (vote_for_ ? vote_for_->get_src_idx() : std::numeric_limits::max()) << "\n"; + } else { + sb << "vote_for=NONE\n"; + } + + if (votes_) { + auto s = votes_->size(); + sb << "votes: "; + std::vector R; + R.resize(desc.get_total_nodes(), -1); + for (td::uint32 i = 0; i < s; i++) { + const auto e = votes_->at(i); + const auto& x = e->get_voters_list(); + for (td::uint32 j = 0; j < desc.get_total_nodes(); j++) { + if (x->at(j)) { + R[j] = e->get_src_idx(); + } + } + } + for (td::uint32 i = 0; i < desc.get_total_nodes(); i++) { + sb << R[i] << " "; + } + sb << "\n"; + } else { + sb << "votes: EMPTY\n"; + } + + sb << "precommits: "; + for (td::uint32 i = 0; i < desc.get_total_nodes(); i++) { + const auto e = precommitted_->at(i); + if (e) { + sb << "+ "; + } else { + sb << "- "; + } + } + sb << "\n"; + sb << "<<<<\n"; +} + + +} // namespace validatorsession + +} // namespace ton diff --git a/validator-session/validator-session-round-attempt-state.h b/validator-session/validator-session-round-attempt-state.h new file mode 100644 index 000000000..a4a6b68bd --- /dev/null +++ b/validator-session/validator-session-round-attempt-state.h @@ -0,0 +1,620 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ + +#pragma once + +#include "td/utils/int_types.h" +#include "td/utils/buffer.h" +#include "adnl/utils.hpp" +#include "common/io.hpp" + +#include "persistent-vector.h" + +#include "validator-session-description.h" + +#include "validator-session-types.h" + +#include + + +namespace ton { + +namespace validatorsession { + +using HashType = ValidatorSessionDescription::HashType; + +struct SessionBlockCandidateSignature : public ValidatorSessionDescription::RootObject { + public: + static auto create_hash(ValidatorSessionDescription& desc, td::Slice data) { + auto obj = create_tl_object(desc.compute_hash(data)); + return desc.compute_hash(serialize_tl_object(obj, true).as_slice()); + } + + static bool compare(const RootObject* r, td::Slice data, HashType hash) { + if (!r || r->get_size() < sizeof(SessionBlockCandidateSignature)) { + return false; + } + auto R = static_cast(r); + return R->hash_ == hash && R->data_.ubegin() == data.ubegin() && R->data_.size() == data.size(); + } + + static auto lookup(ValidatorSessionDescription& desc, td::Slice data, HashType hash, bool temp) { + auto r = desc.get_by_hash(hash, temp); + if (compare(r, data, hash)) { + desc.on_reuse(); + return static_cast(r); + } + return static_cast(nullptr); + } + static SessionBlockCandidateSignature* create(ValidatorSessionDescription& desc, td::BufferSlice value) { + auto hash = create_hash(desc, value.as_slice()); + auto d = static_cast(desc.alloc(value.size(), 8, false)); + td::MutableSlice s{d, value.size()}; + s.copy_from(value.as_slice()); + return new (desc, true) SessionBlockCandidateSignature{desc, s, hash}; + } + static const SessionBlockCandidateSignature* move_to_persistent(ValidatorSessionDescription& desc, + const SessionBlockCandidateSignature* b) { + if (desc.is_persistent(b)) { + return b; + } + CHECK(desc.is_persistent(b->data_.ubegin())); + auto r = lookup(desc, b->data_, b->hash_, false); + if (r) { + return r; + } + return new (desc, false) SessionBlockCandidateSignature{desc, b->data_, b->hash_}; + } + static const SessionBlockCandidateSignature* merge(ValidatorSessionDescription& desc, + const SessionBlockCandidateSignature* l, + const SessionBlockCandidateSignature* r); + SessionBlockCandidateSignature(ValidatorSessionDescription& desc, td::Slice data, HashType hash) + : RootObject(sizeof(SessionBlockCandidateSignature)), data_{data}, hash_(std::move(hash)) { + desc.update_hash(this, hash_); + } + td::BufferSlice value() const { + return td::BufferSlice{data_}; + } + td::Slice as_slice() const { + return data_; + } + auto get_hash(ValidatorSessionDescription& desc) const { + return hash_; + } + + private: + const td::Slice data_; + const HashType hash_; +}; + +using SessionBlockCandidateSignatureVector = CntVector; + +class SentBlock : public ValidatorSessionDescription::RootObject { + public: + static HashType create_hash(ValidatorSessionDescription& desc, td::uint32 src_idx, ValidatorSessionRootHash root_hash, + ValidatorSessionFileHash file_hash, + ValidatorSessionCollatedDataFileHash collated_data_file_hash) { + auto obj = create_tl_object(src_idx, get_vs_hash(desc, root_hash), + get_vs_hash(desc, file_hash), + get_vs_hash(desc, collated_data_file_hash)); + return desc.compute_hash(serialize_tl_object(obj, true).as_slice()); + } + static bool compare(const RootObject* root_object, td::uint32 src_idx, const ValidatorSessionRootHash& root_hash, + const ValidatorSessionFileHash& file_hash, + const ValidatorSessionCollatedDataFileHash& collated_data_file_hash, HashType hash) { + if (!root_object || root_object->get_size() < sizeof(SentBlock)) { + return false; + } + auto obj = static_cast(root_object); + return obj->src_idx_ == src_idx && obj->root_hash_ == root_hash && obj->file_hash_ == file_hash && + obj->collated_data_file_hash_ == collated_data_file_hash && obj->hash_ == hash; + } + static auto lookup(ValidatorSessionDescription& desc, td::uint32 src_idx, const ValidatorSessionRootHash& root_hash, + const ValidatorSessionFileHash& file_hash, + const ValidatorSessionCollatedDataFileHash& collated_data_file_hash, HashType hash, bool temp) { + auto r = desc.get_by_hash(hash, temp); + if (compare(r, src_idx, root_hash, file_hash, collated_data_file_hash, hash)) { + desc.on_reuse(); + return static_cast(r); + } + return static_cast(nullptr); + } + static const SentBlock* create(ValidatorSessionDescription& desc, td::uint32 src_idx, + const ValidatorSessionRootHash& root_hash, const ValidatorSessionFileHash& file_hash, + const ValidatorSessionCollatedDataFileHash& collated_data_file_hash) { + auto hash = create_hash(desc, src_idx, root_hash, file_hash, collated_data_file_hash); + auto r = lookup(desc, src_idx, root_hash, file_hash, collated_data_file_hash, hash, true); + if (r) { + return r; + } + + auto candidate_id = desc.candidate_id(src_idx, root_hash, file_hash, collated_data_file_hash); + + return new (desc, true) SentBlock{desc, src_idx, root_hash, file_hash, collated_data_file_hash, candidate_id, hash}; + } + static const SentBlock* create(ValidatorSessionDescription& desc, const ValidatorSessionCandidateId& zero) { + CHECK(zero.is_zero()); + auto hash = create_hash(desc, 0, ValidatorSessionRootHash::zero(), ValidatorSessionFileHash::zero(), + ValidatorSessionCollatedDataFileHash::zero()); + + return new (desc, true) SentBlock{desc, + 0, + ValidatorSessionRootHash::zero(), + ValidatorSessionFileHash::zero(), + ValidatorSessionCollatedDataFileHash::zero(), + zero, + hash}; + } + static const SentBlock* move_to_persistent(ValidatorSessionDescription& desc, const SentBlock* b) { + if (desc.is_persistent(b)) { + return b; + } + auto r = lookup(desc, b->src_idx_, b->root_hash_, b->file_hash_, b->collated_data_file_hash_, b->hash_, false); + if (r) { + return r; + } + + return new (desc, false) SentBlock{ + desc, b->src_idx_, b->root_hash_, b->file_hash_, b->collated_data_file_hash_, b->candidate_id_, b->hash_}; + } + SentBlock(ValidatorSessionDescription& desc, td::uint32 src_idx, ValidatorSessionRootHash root_hash, + ValidatorSessionFileHash file_hash, ValidatorSessionCollatedDataFileHash collated_data_file_hash, + ValidatorSessionCandidateId candidate_id, HashType hash) + : RootObject(sizeof(SentBlock)) + , src_idx_(src_idx) + , root_hash_(std::move(root_hash)) + , file_hash_(std::move(file_hash)) + , collated_data_file_hash_(std::move(collated_data_file_hash)) + , candidate_id_(candidate_id) + , hash_(std::move(hash)) { + desc.update_hash(this, hash_); + } + auto get_src_idx() const { + return src_idx_; + } + auto get_root_hash() const { + return root_hash_; + } + auto get_file_hash() const { + return file_hash_; + } + auto get_collated_data_file_hash() const { + return collated_data_file_hash_; + } + static ValidatorSessionCandidateId get_block_id(const SentBlock* block) { + return block ? block->candidate_id_ : skip_round_candidate_id(); + } + HashType get_hash(ValidatorSessionDescription& desc) const { + return hash_; + } + bool operator<(const SentBlock& block) const { + if (src_idx_ < block.src_idx_) { + return true; + } + if (src_idx_ > block.src_idx_) { + return false; + } + if (candidate_id_ < block.candidate_id_) { + return true; + } + return false; + } + struct Compare { + bool operator()(const SentBlock* a, const SentBlock* b) const { + return *a < *b; + } + }; + + private: + const td::uint32 src_idx_; + const ValidatorSessionRootHash root_hash_; + const ValidatorSessionFileHash file_hash_; + const ValidatorSessionCollatedDataFileHash collated_data_file_hash_; + const ValidatorSessionCandidateId candidate_id_; + const HashType hash_; +}; + +class SessionBlockCandidate : public ValidatorSessionDescription::RootObject { + public: + static HashType create_hash(ValidatorSessionDescription& desc, HashType block, HashType approved) { + auto obj = create_tl_object(block, approved); + return desc.compute_hash(serialize_tl_object(obj, true).as_slice()); + } + static bool compare(const RootObject* r, const SentBlock* block, const SessionBlockCandidateSignatureVector* approved, + HashType hash) { + if (!r || r->get_size() < sizeof(SessionBlockCandidate)) { + return false; + } + auto R = static_cast(r); + return R->block_ == block && R->approved_by_ == approved && R->hash_ == hash; + } + static auto lookup(ValidatorSessionDescription& desc, const SentBlock* block, + const SessionBlockCandidateSignatureVector* approved, HashType hash, bool temp) { + auto r = desc.get_by_hash(hash, temp); + if (compare(r, block, approved, hash)) { + desc.on_reuse(); + return static_cast(r); + } + return static_cast(nullptr); + } + static const SessionBlockCandidate* create(ValidatorSessionDescription& desc, const SentBlock* block, + const SessionBlockCandidateSignatureVector* approved) { + auto hash = create_hash(desc, get_vs_hash(desc, block), get_vs_hash(desc, approved)); + + auto r = lookup(desc, block, approved, hash, true); + if (r) { + return r; + } + + return new (desc, true) SessionBlockCandidate(desc, block, approved, hash); + } + static const SessionBlockCandidate* create(ValidatorSessionDescription& desc, const SentBlock* block) { + std::vector v; + v.resize(desc.get_total_nodes(), nullptr); + auto vec = SessionBlockCandidateSignatureVector::create(desc, std::move(v)); + return create(desc, block, vec); + } + static const SessionBlockCandidate* move_to_persistent(ValidatorSessionDescription& desc, + const SessionBlockCandidate* b) { + if (desc.is_persistent(b)) { + return b; + } + auto block = SentBlock::move_to_persistent(desc, b->block_); + auto approved = SessionBlockCandidateSignatureVector::move_to_persistent(desc, b->approved_by_); + auto r = lookup(desc, block, approved, b->hash_, false); + if (r) { + return r; + } + + return new (desc, false) SessionBlockCandidate{desc, block, approved, b->hash_}; + } + SessionBlockCandidate(ValidatorSessionDescription& desc, const SentBlock* block, + const SessionBlockCandidateSignatureVector* approved, HashType hash) + : RootObject{sizeof(SessionBlockCandidate)}, block_(block), approved_by_(approved), hash_(hash) { + desc.update_hash(this, hash_); + } + static const SessionBlockCandidate* merge(ValidatorSessionDescription& desc, const SessionBlockCandidate* l, + const SessionBlockCandidate* r); + auto get_block() const { + return block_; + } + auto get_id() const { + return SentBlock::get_block_id(block_); + } + auto get_src_idx() const { + return block_ ? block_->get_src_idx() : std::numeric_limits::max(); + } + bool check_block_is_approved_by(td::uint32 src_idx) const { + return approved_by_->at(src_idx); + } + bool check_block_is_approved(ValidatorSessionDescription& desc) const; + auto get_hash(ValidatorSessionDescription& desc) const { + return hash_; + } + auto get_approvers_list() const { + return approved_by_; + } + static const SessionBlockCandidate* push(ValidatorSessionDescription& desc, const SessionBlockCandidate* state, + td::uint32 src_idx, const SessionBlockCandidateSignature* sig) { + CHECK(state); + if (state->approved_by_->at(src_idx)) { + return state; + } + return create(desc, state->block_, + SessionBlockCandidateSignatureVector::change(desc, state->approved_by_, src_idx, sig)); + } + class Compare { + public: + bool operator()(const SessionBlockCandidate* l, const SessionBlockCandidate* r) { + return l->get_id() < r->get_id(); + } + }; + + private: + const SentBlock* block_; + const SessionBlockCandidateSignatureVector* approved_by_; + const HashType hash_; +}; + +class SessionVoteCandidate : public ValidatorSessionDescription::RootObject { + public: + static HashType create_hash(ValidatorSessionDescription& desc, HashType block, HashType voted) { + auto obj = create_tl_object(block, voted); + return desc.compute_hash(serialize_tl_object(obj, true).as_slice()); + } + static bool compare(const RootObject* r, const SentBlock* block, const CntVector* voted, HashType hash) { + if (!r || r->get_size() < sizeof(SessionVoteCandidate)) { + return false; + } + auto R = static_cast(r); + return R->block_ == block && R->voted_by_ == voted && R->hash_ == hash; + } + static auto lookup(ValidatorSessionDescription& desc, const SentBlock* block, const CntVector* voted, + HashType hash, bool temp) { + auto r = desc.get_by_hash(hash, temp); + if (compare(r, block, voted, hash)) { + desc.on_reuse(); + return static_cast(r); + } + return static_cast(nullptr); + } + static const SessionVoteCandidate* create(ValidatorSessionDescription& desc, const SentBlock* block, + const CntVector* voted) { + auto hash = create_hash(desc, get_vs_hash(desc, block), get_vs_hash(desc, voted)); + + auto r = lookup(desc, block, voted, hash, true); + if (r) { + return r; + } + + return new (desc, true) SessionVoteCandidate(desc, block, voted, hash); + } + static const SessionVoteCandidate* create(ValidatorSessionDescription& desc, const SentBlock* block) { + std::vector v; + v.resize(desc.get_total_nodes(), false); + auto vec = CntVector::create(desc, std::move(v)); + return create(desc, block, vec); + } + static const SessionVoteCandidate* move_to_persistent(ValidatorSessionDescription& desc, + const SessionVoteCandidate* b) { + if (desc.is_persistent(b)) { + return b; + } + auto block = SentBlock::move_to_persistent(desc, b->block_); + auto voted = CntVector::move_to_persistent(desc, b->voted_by_); + auto r = lookup(desc, block, voted, b->hash_, false); + if (r) { + return r; + } + + return new (desc, false) SessionVoteCandidate{desc, block, voted, b->hash_}; + } + SessionVoteCandidate(ValidatorSessionDescription& desc, const SentBlock* block, const CntVector* voted, + HashType hash) + : RootObject{sizeof(SessionVoteCandidate)}, block_(block), voted_by_(voted), hash_(hash) { + desc.update_hash(this, hash_); + } + static const SessionVoteCandidate* merge(ValidatorSessionDescription& desc, const SessionVoteCandidate* l, + const SessionVoteCandidate* r); + auto get_block() const { + return block_; + } + auto get_id() const { + return SentBlock::get_block_id(block_); + } + auto get_src_idx() const { + return block_ ? block_->get_src_idx() : std::numeric_limits::max(); + } + bool check_block_is_voted_by(td::uint32 src_idx) const { + return voted_by_->at(src_idx); + } + bool check_block_is_voted(ValidatorSessionDescription& desc) const; + auto get_hash(ValidatorSessionDescription& desc) const { + return hash_; + } + auto get_voters_list() const { + return voted_by_; + } + static const SessionVoteCandidate* push(ValidatorSessionDescription& desc, const SessionVoteCandidate* state, + td::uint32 src_idx) { + CHECK(state); + if (state->voted_by_->at(src_idx)) { + return state; + } + return create(desc, state->block_, CntVector::change(desc, state->voted_by_, src_idx, true)); + } + class Compare { + public: + bool operator()(const SessionVoteCandidate* l, const SessionVoteCandidate* r) { + return l->get_id() < r->get_id(); + } + }; + + private: + const SentBlock* block_; + const CntVector* voted_by_; + const HashType hash_; +}; + +using VoteVector = CntSortedVector; +using ApproveVector = CntSortedVector; +class ValidatorSessionRoundState; + +class ValidatorSessionRoundAttemptState : public ValidatorSessionDescription::RootObject { + public: + static HashType create_hash(ValidatorSessionDescription& desc, td::uint32 seqno, HashType votes, + HashType precommitted, bool vote_for_inited, HashType vote_for) { + auto obj = create_tl_object(seqno, votes, precommitted, + vote_for_inited, vote_for); + return desc.compute_hash(serialize_tl_object(obj, true).as_slice()); + } + static bool compare(const RootObject* r, td::uint32 seqno, const VoteVector* votes, + const CntVector* precommitted, const SentBlock* vote_for, bool vote_for_inited, + HashType hash) { + if (!r || r->get_size() < sizeof(ValidatorSessionRoundAttemptState)) { + return false; + } + auto R = static_cast(r); + return R->seqno_ == seqno && R->votes_ == votes && R->precommitted_ == precommitted && R->vote_for_ == vote_for && + R->vote_for_inited_ == vote_for_inited && R->hash_ == hash; + } + static auto lookup(ValidatorSessionDescription& desc, td::uint32 seqno, const VoteVector* votes, + const CntVector* precommitted, const SentBlock* vote_for, bool vote_for_inited, + HashType hash, bool temp) { + auto r = desc.get_by_hash(hash, temp); + if (compare(r, seqno, votes, precommitted, vote_for, vote_for_inited, hash)) { + desc.on_reuse(); + return static_cast(r); + } + return static_cast(nullptr); + } + static const ValidatorSessionRoundAttemptState* move_to_persistent(ValidatorSessionDescription& desc, + const ValidatorSessionRoundAttemptState* b) { + if (desc.is_persistent(b)) { + return b; + } + auto votes = VoteVector::move_to_persistent(desc, b->votes_); + auto precommitted = CntVector::move_to_persistent(desc, b->precommitted_); + auto vote_for = SentBlock::move_to_persistent(desc, b->vote_for_); + + auto r = lookup(desc, b->seqno_, votes, precommitted, vote_for, b->vote_for_inited_, b->hash_, false); + if (r) { + return r; + } + + return new (desc, false) ValidatorSessionRoundAttemptState{desc, b->seqno_, votes, precommitted, + vote_for, b->vote_for_inited_, b->hash_}; + } + + static const ValidatorSessionRoundAttemptState* create(ValidatorSessionDescription& desc, td::uint32 seqno, + const VoteVector* votes, const CntVector* precommitted, + const SentBlock* vote_for, bool vote_for_inited) { + auto hash = create_hash(desc, seqno, get_vs_hash(desc, votes), get_vs_hash(desc, precommitted), + get_vs_hash(desc, vote_for), vote_for_inited); + + auto r = lookup(desc, seqno, votes, precommitted, vote_for, vote_for_inited, hash, true); + if (r) { + return r; + } + + return new (desc, true) + ValidatorSessionRoundAttemptState(desc, seqno, votes, precommitted, vote_for, vote_for_inited, hash); + } + static const ValidatorSessionRoundAttemptState* create(ValidatorSessionDescription& desc, td::uint32 seqno) { + std::vector x; + x.resize(desc.get_total_nodes(), false); + auto p = CntVector::create(desc, std::move(x)); + + return create(desc, seqno, nullptr, p, nullptr, false); + } + + ValidatorSessionRoundAttemptState(ValidatorSessionDescription& desc, td::uint32 seqno, const VoteVector* votes, + const CntVector* precommitted, const SentBlock* vote_for, + bool vote_for_inited, HashType hash) + : RootObject{sizeof(ValidatorSessionRoundAttemptState)} + , seqno_(seqno) + , votes_(votes) + , precommitted_(precommitted) + , vote_for_(vote_for) + , vote_for_inited_(vote_for_inited) + , hash_(std::move(hash)) { + desc.update_hash(this, hash_); + } + auto get_hash(ValidatorSessionDescription& desc) const { + return hash_; + } + auto get_seqno() const { + return seqno_; + } + auto get_votes() const { + return votes_; + } + auto get_precommits() const { + return precommitted_; + } + const SentBlock* get_voted_block(ValidatorSessionDescription& desc, bool& f) const; + const SentBlock* get_vote_for_block(ValidatorSessionDescription& desc, bool& f) const { + f = vote_for_inited_; + return vote_for_; + } + bool check_attempt_is_precommitted(ValidatorSessionDescription& desc) const; + bool check_vote_received_from(td::uint32 src_idx) const; + bool check_precommit_received_from(td::uint32 src_idx) const; + + bool operator<(const ValidatorSessionRoundAttemptState& right) const { + return seqno_ < right.seqno_; + } + struct Compare { + bool operator()(const ValidatorSessionRoundAttemptState* a, const ValidatorSessionRoundAttemptState* b) const { + return *a < *b; + } + }; + + static const ValidatorSessionRoundAttemptState* merge(ValidatorSessionDescription& desc, + const ValidatorSessionRoundAttemptState* left, + const ValidatorSessionRoundAttemptState* right); + static const ValidatorSessionRoundAttemptState* action(ValidatorSessionDescription& desc, + const ValidatorSessionRoundAttemptState* state, + td::uint32 src_idx, td::uint32 att, + const ton_api::validatorSession_message_voteFor& act, + const ValidatorSessionRoundState* round); + static const ValidatorSessionRoundAttemptState* action(ValidatorSessionDescription& desc, + const ValidatorSessionRoundAttemptState* state, + td::uint32 src_idx, td::uint32 att, + const ton_api::validatorSession_message_vote& act, + const ValidatorSessionRoundState* round); + static const ValidatorSessionRoundAttemptState* action(ValidatorSessionDescription& desc, + const ValidatorSessionRoundAttemptState* state, + td::uint32 src_idx, td::uint32 att, + const ton_api::validatorSession_message_precommit& act, + const ValidatorSessionRoundState* round); + static const ValidatorSessionRoundAttemptState* action(ValidatorSessionDescription& desc, + const ValidatorSessionRoundAttemptState* state, + td::uint32 src_idx, td::uint32 att, + const ton_api::validatorSession_message_empty& act, + const ValidatorSessionRoundState* round); + static const ValidatorSessionRoundAttemptState* action(ValidatorSessionDescription& desc, + const ValidatorSessionRoundAttemptState* state, + td::uint32 src_idx, td::uint32 att, + const ton_api::validatorSession_round_Message* action, + const ValidatorSessionRoundState* round); + template + static const ValidatorSessionRoundAttemptState* action(ValidatorSessionDescription& desc, + const ValidatorSessionRoundAttemptState* state, + td::uint32 src_idx, td::uint32 att, const T& action, + const ValidatorSessionRoundState* round) { + UNREACHABLE(); + } + static const ValidatorSessionRoundAttemptState* try_vote(ValidatorSessionDescription& desc, + const ValidatorSessionRoundAttemptState* state, + td::uint32 src_idx, td::uint32 att, + const ValidatorSessionRoundState* round, + const ton_api::validatorSession_round_Message* cmp, + bool& made); + static const ValidatorSessionRoundAttemptState* try_precommit(ValidatorSessionDescription& desc, + const ValidatorSessionRoundAttemptState* state, + td::uint32 src_idx, td::uint32 att, + const ValidatorSessionRoundState* round, + const ton_api::validatorSession_round_Message* cmp, + bool& made); + static const ValidatorSessionRoundAttemptState* make_one(ValidatorSessionDescription& desc, + const ValidatorSessionRoundAttemptState* state, + td::uint32 src_idx, td::uint32 att, + const ValidatorSessionRoundState* round, + const ton_api::validatorSession_round_Message* cmp, + bool& made); + tl_object_ptr create_action(ValidatorSessionDescription& desc, + const ValidatorSessionRoundState* round, + td::uint32 src_idx, td::uint32 att) const; + void dump(ValidatorSessionDescription& desc, td::StringBuilder& sb) const; + + private: + const td::uint32 seqno_; + const VoteVector* votes_; + const CntVector* precommitted_; + const SentBlock* vote_for_; + const bool vote_for_inited_; + const HashType hash_; +}; + +using AttemptVector = + CntSortedVector; + + +} // namespace validatorsession + +} // namespace ton diff --git a/validator-session/validator-session-state.cpp b/validator-session/validator-session-state.cpp index 62a054d54..97b918954 100644 --- a/validator-session/validator-session-state.cpp +++ b/validator-session/validator-session-state.cpp @@ -67,14 +67,6 @@ namespace ton { namespace validatorsession { -static td::uint32 get_round_id(const ton_api::validatorSession_round_Message* message) { - td::uint32 round = 0; - bool is_called = ton_api::downcast_call(*const_cast(message), - [&](auto& obj) { round = obj.round_; }); - CHECK(is_called); - return round; -} - static const ValidatorSessionRoundAttemptState* get_attempt(const AttemptVector* vec, td::uint32 seqno) { if (!vec) { return nullptr; @@ -108,118 +100,12 @@ static const SessionBlockCandidate* get_candidate(const ApproveVector* vec, Vali return nullptr; } -static const SessionVoteCandidate* get_candidate(const VoteVector* vec, ValidatorSessionCandidateId id) { - if (!vec) { - return nullptr; - } - auto size = vec->size(); - auto v = vec->data(); - for (td::uint32 i = 0; i < size; i++) { - if (v[i]->get_id() == id) { - return v[i]; - } - } - return nullptr; -} - -// -// -// SessionBlockCandidateSignature -// -// - -const SessionBlockCandidateSignature* SessionBlockCandidateSignature::merge(ValidatorSessionDescription& desc, - const SessionBlockCandidateSignature* l, - const SessionBlockCandidateSignature* r) { - if (!l) { - return r; - } - if (!r) { - return l; - } - if (l == r) { - return l; - } - if (l->as_slice() < r->as_slice()) { - return l; - } else { - return r; - } -} - -// -// -// SessionBlockCandidate -// -// - -bool SessionBlockCandidate::check_block_is_approved(ValidatorSessionDescription& desc) const { - ValidatorWeight w = 0; - for (td::uint32 i = 0; i < desc.get_total_nodes(); i++) { - if (approved_by_->at(i)) { - w += desc.get_node_weight(i); - if (w >= desc.get_cutoff_weight()) { - return true; - } - } - } - return false; -} - -const SessionBlockCandidate* SessionBlockCandidate::merge(ValidatorSessionDescription& desc, - const SessionBlockCandidate* l, - const SessionBlockCandidate* r) { - if (!l) { - return r; - } - if (!r) { - return l; - } - if (l == r) { - return l; - } - CHECK(l->get_id() == r->get_id()); - auto v = SessionBlockCandidateSignatureVector::merge( - desc, l->approved_by_, r->approved_by_, - [&](const SessionBlockCandidateSignature* l, const SessionBlockCandidateSignature* r) { - return SessionBlockCandidateSignature::merge(desc, l, r); - }); - return SessionBlockCandidate::create(desc, l->block_, std::move(v)); -} - -// -// -// SessionVoteCandidate -// -// - -bool SessionVoteCandidate::check_block_is_voted(ValidatorSessionDescription& desc) const { - ValidatorWeight w = 0; - for (td::uint32 i = 0; i < desc.get_total_nodes(); i++) { - if (voted_by_->at(i)) { - w += desc.get_node_weight(i); - if (w >= desc.get_cutoff_weight()) { - return true; - } - } - } - return false; -} - -const SessionVoteCandidate* SessionVoteCandidate::merge(ValidatorSessionDescription& desc, - const SessionVoteCandidate* l, const SessionVoteCandidate* r) { - if (!l) { - return r; - } - if (!r) { - return l; - } - if (l == r) { - return l; - } - CHECK(l->get_id() == r->get_id()); - auto v = CntVector::merge(desc, l->voted_by_, r->voted_by_); - return SessionVoteCandidate::create(desc, l->block_, std::move(v)); +static td::uint32 get_round_id(const ton_api::validatorSession_round_Message* message) { + td::uint32 round = 0; + bool is_called = ton_api::downcast_call(*const_cast(message), + [&](auto& obj) { round = obj.round_; }); + CHECK(is_called); + return round; } // @@ -374,346 +260,6 @@ const ValidatorSessionOldRoundState* ValidatorSessionOldRoundState::action( return state; } -// -// -// ATTEMPT STATE -// -// - -const ValidatorSessionRoundAttemptState* ValidatorSessionRoundAttemptState::merge( - ValidatorSessionDescription& desc, const ValidatorSessionRoundAttemptState* left, - const ValidatorSessionRoundAttemptState* right) { - if (!left) { - return right; - } - if (!right) { - return left; - } - if (left == right) { - return left; - } - CHECK(left->seqno_ == right->seqno_); - - const SentBlock* vote_for = nullptr; - bool vote_for_inited = false; - if (!left->vote_for_inited_) { - vote_for = right->vote_for_; - vote_for_inited = right->vote_for_inited_; - } else if (!right->vote_for_inited_) { - vote_for = left->vote_for_; - vote_for_inited = left->vote_for_inited_; - } else if (left->vote_for_ == right->vote_for_) { - vote_for_inited = true; - vote_for = left->vote_for_; - } else { - auto l = SentBlock::get_block_id(left->vote_for_); - auto r = SentBlock::get_block_id(right->vote_for_); - - vote_for_inited = true; - if (l < r) { - vote_for = left->vote_for_; - } else { - vote_for = right->vote_for_; - } - } - - auto precommitted = CntVector::merge(desc, left->precommitted_, right->precommitted_); - auto votes = VoteVector::merge(desc, left->votes_, right->votes_, - [&](const SessionVoteCandidate* l, const SessionVoteCandidate* r) { - return SessionVoteCandidate::merge(desc, l, r); - }); - - return ValidatorSessionRoundAttemptState::create(desc, left->seqno_, std::move(votes), std::move(precommitted), - vote_for, vote_for_inited); -} - -const ValidatorSessionRoundAttemptState* ValidatorSessionRoundAttemptState::action( - ValidatorSessionDescription& desc, const ValidatorSessionRoundAttemptState* state, td::uint32 src_idx, - td::uint32 att, const ton_api::validatorSession_message_voteFor& act, const ValidatorSessionRoundState* round) { - if (state->vote_for_inited_) { - VLOG(VALIDATOR_SESSION_WARNING) << "[validator session][node " << desc.get_source_id(src_idx) << "][" << act - << "]: invalid message: duplicate VOTEFOR"; - return state; - } - if (src_idx != desc.get_vote_for_author(att)) { - VLOG(VALIDATOR_SESSION_WARNING) << "[validator session][node " << desc.get_source_id(src_idx) << "][" << act - << "]: invalid message: bad VOTEFOR author"; - return state; - } - if (round->get_first_attempt(src_idx) == 0 && desc.opts().max_round_attempts > 0) { - VLOG(VALIDATOR_SESSION_WARNING) << "[validator session][node " << desc.get_source_id(src_idx) << "][" << act - << "]: invalid message: too early for VOTEFOR"; - return state; - } - if (round->get_first_attempt(src_idx) + desc.opts().max_round_attempts > att && desc.opts().max_round_attempts == 0) { - VLOG(VALIDATOR_SESSION_WARNING) << "[validator session][node " << desc.get_source_id(src_idx) << "][" << act - << "]: invalid message: too early for VOTEFOR"; - return state; - } - - auto x = round->get_block(act.candidate_); - - if (!x) { - VLOG(VALIDATOR_SESSION_WARNING) << "[validator session][node " << desc.get_source_id(src_idx) << "][" << act - << "]: invalid message: VOTEFOR for not approved block"; - return state; - } - if (!x->check_block_is_approved(desc)) { - VLOG(VALIDATOR_SESSION_WARNING) << "[validator session][node " << desc.get_source_id(src_idx) << "][" << act - << "]: invalid message: VOTEFOR for not approved block"; - return state; - } - - return ValidatorSessionRoundAttemptState::create(desc, state->seqno_, state->votes_, state->precommitted_, - x->get_block(), true); -} - -const ValidatorSessionRoundAttemptState* ValidatorSessionRoundAttemptState::action( - ValidatorSessionDescription& desc, const ValidatorSessionRoundAttemptState* state, td::uint32 src_idx, - td::uint32 att, const ton_api::validatorSession_message_vote& act, const ValidatorSessionRoundState* round) { - bool made; - return make_one(desc, state, src_idx, att, round, &act, made); -} - -const ValidatorSessionRoundAttemptState* ValidatorSessionRoundAttemptState::action( - ValidatorSessionDescription& desc, const ValidatorSessionRoundAttemptState* state, td::uint32 src_idx, - td::uint32 att, const ton_api::validatorSession_message_precommit& act, const ValidatorSessionRoundState* round) { - bool made; - return make_one(desc, state, src_idx, att, round, &act, made); -} - -const ValidatorSessionRoundAttemptState* ValidatorSessionRoundAttemptState::action( - ValidatorSessionDescription& desc, const ValidatorSessionRoundAttemptState* state, td::uint32 src_idx, - td::uint32 att, const ton_api::validatorSession_message_empty& act, const ValidatorSessionRoundState* round) { - bool made; - return make_one(desc, state, src_idx, att, round, &act, made); -} - -const ValidatorSessionRoundAttemptState* ValidatorSessionRoundAttemptState::try_vote( - ValidatorSessionDescription& desc, const ValidatorSessionRoundAttemptState* state, td::uint32 src_idx, - td::uint32 att, const ValidatorSessionRoundState* round, const ton_api::validatorSession_round_Message* act, - bool& made) { - made = false; - if (state->check_vote_received_from(src_idx)) { - return state; - } - auto found = false; - auto block = round->choose_block_to_vote(desc, src_idx, att, state->vote_for_, state->vote_for_inited_, found); - if (!found) { - return state; - } - auto block_id = SentBlock::get_block_id(block); - made = true; - - if (act) { - if (act->get_id() != ton_api::validatorSession_message_vote::ID) { - VLOG(VALIDATOR_SESSION_WARNING) << "[validator session][node " << desc.get_source_id(src_idx) << "][" << act - << "]: expected VOTE(" << block_id << ")"; - } else { - auto x = static_cast(act); - if (x->candidate_ != block_id) { - VLOG(VALIDATOR_SESSION_WARNING) << "[validator session][node " << desc.get_source_id(src_idx) << "][" << act - << "]: expected VOTE(" << block_id << ")"; - } - } - } else { - VLOG(VALIDATOR_SESSION_WARNING) << "[validator session][node " << desc.get_source_id(src_idx) - << "]: making implicit VOTE(" << block_id << ")"; - } - - auto candidate = get_candidate(state->votes_, block_id); - if (!candidate) { - candidate = SessionVoteCandidate::create(desc, block); - } - candidate = SessionVoteCandidate::push(desc, candidate, src_idx); - auto v = VoteVector::push(desc, state->votes_, candidate); - return ValidatorSessionRoundAttemptState::create(desc, state->seqno_, std::move(v), state->precommitted_, - state->vote_for_, state->vote_for_inited_); -} - -const ValidatorSessionRoundAttemptState* ValidatorSessionRoundAttemptState::try_precommit( - ValidatorSessionDescription& desc, const ValidatorSessionRoundAttemptState* state, td::uint32 src_idx, - td::uint32 att, const ValidatorSessionRoundState* round, const ton_api::validatorSession_round_Message* act, - bool& made) { - made = false; - if (state->check_precommit_received_from(src_idx)) { - return state; - } - bool found; - auto block = state->get_voted_block(desc, found); - if (!found) { - return state; - } - made = true; - auto block_id = SentBlock::get_block_id(block); - - if (act) { - if (act->get_id() != ton_api::validatorSession_message_precommit::ID) { - VLOG(VALIDATOR_SESSION_WARNING) << "[validator session][node " << desc.get_source_id(src_idx) << "][" << act - << "]: expected PRECOMMIT(" << block_id << ")"; - } else { - auto x = static_cast(act); - if (x->candidate_ != block_id) { - VLOG(VALIDATOR_SESSION_WARNING) << "[validator session][node " << desc.get_source_id(src_idx) << "][" << act - << "]: expected PRECOMMIT(" << block_id << ")"; - } - } - } else { - VLOG(VALIDATOR_SESSION_WARNING) << "[validator session][node " << desc.get_source_id(src_idx) - << "]: making implicit PRECOMMIT(" << block_id << ")"; - } - - auto v = CntVector::change(desc, state->precommitted_, src_idx, true); - return ValidatorSessionRoundAttemptState::create(desc, state->seqno_, state->votes_, std::move(v), state->vote_for_, - state->vote_for_inited_); -} - -const ValidatorSessionRoundAttemptState* ValidatorSessionRoundAttemptState::make_one( - ValidatorSessionDescription& desc, const ValidatorSessionRoundAttemptState* state, td::uint32 src_idx, - td::uint32 att, const ValidatorSessionRoundState* round, const ton_api::validatorSession_round_Message* act, - bool& made) { - made = false; - state = try_vote(desc, state, src_idx, att, round, act, made); - if (made) { - return state; - } - state = try_precommit(desc, state, src_idx, att, round, act, made); - if (made) { - return state; - } - if (act && act->get_id() != ton_api::validatorSession_message_empty::ID) { - VLOG(VALIDATOR_SESSION_WARNING) << "[validator session][node " << desc.get_source_id(src_idx) << "][" << act - << "]: invalid message: expected EMPTY"; - } - return state; -} - -const ValidatorSessionRoundAttemptState* ValidatorSessionRoundAttemptState::action( - ValidatorSessionDescription& desc, const ValidatorSessionRoundAttemptState* state, td::uint32 src_idx, - td::uint32 att, const ton_api::validatorSession_round_Message* act, const ValidatorSessionRoundState* round) { - ton_api::downcast_call(*const_cast(act), - [&](auto& obj) { state = action(desc, state, src_idx, att, obj, round); }); - return state; -} - -bool ValidatorSessionRoundAttemptState::check_vote_received_from(td::uint32 src_idx) const { - if (!votes_) { - return false; - } - auto size = votes_->size(); - auto v = votes_->data(); - for (td::uint32 i = 0; i < size; i++) { - if (v[i]->check_block_is_voted_by(src_idx)) { - return true; - } - } - return false; -} - -bool ValidatorSessionRoundAttemptState::check_precommit_received_from(td::uint32 src_idx) const { - return precommitted_->at(src_idx); -} - -const SentBlock* ValidatorSessionRoundAttemptState::get_voted_block(ValidatorSessionDescription& desc, bool& f) const { - f = false; - if (!votes_) { - return nullptr; - } - - auto size = votes_->size(); - auto v = votes_->data(); - for (td::uint32 i = 0; i < size; i++) { - if (v[i]->check_block_is_voted(desc)) { - f = true; - return v[i]->get_block(); - } - } - return nullptr; -} - -bool ValidatorSessionRoundAttemptState::check_attempt_is_precommitted(ValidatorSessionDescription& desc) const { - ValidatorWeight weight = 0; - for (td::uint32 i = 0; i < desc.get_total_nodes(); i++) { - if (precommitted_->at(i)) { - weight += desc.get_node_weight(i); - if (weight >= desc.get_cutoff_weight()) { - return true; - } - } - } - return false; -} - -tl_object_ptr ValidatorSessionRoundAttemptState::create_action( - ValidatorSessionDescription& desc, const ValidatorSessionRoundState* round, td::uint32 src_idx, - td::uint32 att) const { - if (!check_vote_received_from(src_idx)) { - auto found = false; - auto B = round->choose_block_to_vote(desc, src_idx, att, vote_for_, vote_for_inited_, found); - if (found) { - auto block_id = SentBlock::get_block_id(B); - return create_tl_object(round->get_seqno(), seqno_, block_id); - } - } - if (!check_precommit_received_from(src_idx)) { - bool f = false; - auto B = get_voted_block(desc, f); - - if (f) { - auto block_id = SentBlock::get_block_id(B); - - return create_tl_object(round->get_seqno(), seqno_, block_id); - } - } - - return create_tl_object(round->get_seqno(), seqno_); -} - -void ValidatorSessionRoundAttemptState::dump(ValidatorSessionDescription& desc, td::StringBuilder& sb) const { - sb << "attempt=" << seqno_ << "\n"; - sb << ">>>>\n"; - - if (vote_for_inited_) { - sb << "vote_for=" << (vote_for_ ? vote_for_->get_src_idx() : std::numeric_limits::max()) << "\n"; - } else { - sb << "vote_for=NONE\n"; - } - - if (votes_) { - auto s = votes_->size(); - sb << "votes: "; - std::vector R; - R.resize(desc.get_total_nodes(), -1); - for (td::uint32 i = 0; i < s; i++) { - const auto e = votes_->at(i); - const auto& x = e->get_voters_list(); - for (td::uint32 j = 0; j < desc.get_total_nodes(); j++) { - if (x->at(j)) { - R[j] = e->get_src_idx(); - } - } - } - for (td::uint32 i = 0; i < desc.get_total_nodes(); i++) { - sb << R[i] << " "; - } - sb << "\n"; - } else { - sb << "votes: EMPTY\n"; - } - - sb << "precommits: "; - for (td::uint32 i = 0; i < desc.get_total_nodes(); i++) { - const auto e = precommitted_->at(i); - if (e) { - sb << "+ "; - } else { - sb << "- "; - } - } - sb << "\n"; - sb << "<<<<\n"; -} - // // // ROUND STATE @@ -821,8 +367,6 @@ const ValidatorSessionRoundState* ValidatorSessionRoundState::merge(ValidatorSes return SessionBlockCandidateSignature::merge(desc, l, r); }); - //auto sent_vec = SentBlockVector::merge(desc, left->sent_blocks_, right->sent_blocks_); - auto sent = ApproveVector::merge(desc, left->sent_blocks_, right->sent_blocks_, [&](const SessionBlockCandidate* l, const SessionBlockCandidate* r) { return SessionBlockCandidate::merge(desc, l, r); @@ -985,9 +529,6 @@ const ValidatorSessionRoundState* ValidatorSessionRoundState::forward_action_to_ attempt = ValidatorSessionRoundAttemptState::create(desc, att); } - bool had_voted_block; - attempt->get_voted_block(desc, had_voted_block); - ton_api::downcast_call(*const_cast(act), [&](auto& obj) { attempt = ValidatorSessionRoundAttemptState::action(desc, attempt, src_idx, att, obj, state); }); @@ -1302,6 +843,8 @@ std::vector ValidatorSessionRoundState::choose_blocks_to_appro CHECK(prio >= 0); td::uint32 blk_src_idx = B->get_src_idx(); if (was_source.count(blk_src_idx) > 0) { + // Any honest validator submits at most one block in a round + // Therefore, we can ignore all blocks from a node if it submits more than one x[prio] = nullptr; } else { was_source.insert(blk_src_idx); diff --git a/validator-session/validator-session-state.h b/validator-session/validator-session-state.h index 37708cba9..f827b3fcb 100644 --- a/validator-session/validator-session-state.h +++ b/validator-session/validator-session-state.h @@ -24,15 +24,15 @@ #include "common/io.hpp" #include "persistent-vector.h" - #include "validator-session-description.h" - #include "validator-session-types.h" +#include "validator-session-round-attempt-state.h" #include namespace td { +td::StringBuilder& operator<<(td::StringBuilder& sb, const ton::ton_api::validatorSession_round_Message& message); td::StringBuilder& operator<<(td::StringBuilder& sb, const ton::ton_api::validatorSession_round_Message* message); } @@ -41,410 +41,6 @@ namespace ton { namespace validatorsession { -using HashType = ValidatorSessionDescription::HashType; - -constexpr int VERBOSITY_NAME(VALIDATOR_SESSION_WARNING) = verbosity_WARNING; -constexpr int VERBOSITY_NAME(VALIDATOR_SESSION_NOTICE) = verbosity_DEBUG; -constexpr int VERBOSITY_NAME(VALIDATOR_SESSION_INFO) = verbosity_DEBUG; -constexpr int VERBOSITY_NAME(VALIDATOR_SESSION_DEBUG) = verbosity_DEBUG; -constexpr int VERBOSITY_NAME(VALIDATOR_SESSION_EXTRA_DEBUG) = verbosity_DEBUG + 1; - -struct SessionBlockCandidateSignature : public ValidatorSessionDescription::RootObject { - public: - static auto create_hash(ValidatorSessionDescription& desc, td::Slice data) { - auto obj = create_tl_object(desc.compute_hash(data)); - return desc.compute_hash(serialize_tl_object(obj, true).as_slice()); - } - - static bool compare(const RootObject* r, td::Slice data, HashType hash) { - if (!r || r->get_size() < sizeof(SessionBlockCandidateSignature)) { - return false; - } - auto R = static_cast(r); - return R->hash_ == hash && R->data_.ubegin() == data.ubegin() && R->data_.size() == data.size(); - } - - static auto lookup(ValidatorSessionDescription& desc, td::Slice data, HashType hash, bool temp) { - auto r = desc.get_by_hash(hash, temp); - if (compare(r, data, hash)) { - desc.on_reuse(); - return static_cast(r); - } - return static_cast(nullptr); - } - static SessionBlockCandidateSignature* create(ValidatorSessionDescription& desc, td::BufferSlice value) { - auto hash = create_hash(desc, value.as_slice()); - auto d = static_cast(desc.alloc(value.size(), 8, false)); - td::MutableSlice s{d, value.size()}; - s.copy_from(value.as_slice()); - return new (desc, true) SessionBlockCandidateSignature{desc, s, hash}; - } - static const SessionBlockCandidateSignature* move_to_persistent(ValidatorSessionDescription& desc, - const SessionBlockCandidateSignature* b) { - if (desc.is_persistent(b)) { - return b; - } - CHECK(desc.is_persistent(b->data_.ubegin())); - auto r = lookup(desc, b->data_, b->hash_, false); - if (r) { - return r; - } - return new (desc, false) SessionBlockCandidateSignature{desc, b->data_, b->hash_}; - } - static const SessionBlockCandidateSignature* merge(ValidatorSessionDescription& desc, - const SessionBlockCandidateSignature* l, - const SessionBlockCandidateSignature* r); - SessionBlockCandidateSignature(ValidatorSessionDescription& desc, td::Slice data, HashType hash) - : RootObject(sizeof(SessionBlockCandidateSignature)), data_{data}, hash_(std::move(hash)) { - desc.update_hash(this, hash_); - } - td::BufferSlice value() const { - return td::BufferSlice{data_}; - } - td::Slice as_slice() const { - return data_; - } - auto get_hash(ValidatorSessionDescription& desc) const { - return hash_; - } - - private: - const td::Slice data_; - const HashType hash_; -}; - -using SessionBlockCandidateSignatureVector = CntVector; - -class SentBlock : public ValidatorSessionDescription::RootObject { - public: - static HashType create_hash(ValidatorSessionDescription& desc, td::uint32 src_idx, ValidatorSessionRootHash root_hash, - ValidatorSessionFileHash file_hash, - ValidatorSessionCollatedDataFileHash collated_data_file_hash) { - auto obj = create_tl_object(src_idx, get_vs_hash(desc, root_hash), - get_vs_hash(desc, file_hash), - get_vs_hash(desc, collated_data_file_hash)); - return desc.compute_hash(serialize_tl_object(obj, true).as_slice()); - } - static bool compare(const RootObject* root_object, td::uint32 src_idx, const ValidatorSessionRootHash& root_hash, - const ValidatorSessionFileHash& file_hash, - const ValidatorSessionCollatedDataFileHash& collated_data_file_hash, HashType hash) { - if (!root_object || root_object->get_size() < sizeof(SentBlock)) { - return false; - } - auto obj = static_cast(root_object); - return obj->src_idx_ == src_idx && obj->root_hash_ == root_hash && obj->file_hash_ == file_hash && - obj->collated_data_file_hash_ == collated_data_file_hash && obj->hash_ == hash; - } - static auto lookup(ValidatorSessionDescription& desc, td::uint32 src_idx, const ValidatorSessionRootHash& root_hash, - const ValidatorSessionFileHash& file_hash, - const ValidatorSessionCollatedDataFileHash& collated_data_file_hash, HashType hash, bool temp) { - auto r = desc.get_by_hash(hash, temp); - if (compare(r, src_idx, root_hash, file_hash, collated_data_file_hash, hash)) { - desc.on_reuse(); - return static_cast(r); - } - return static_cast(nullptr); - } - static const SentBlock* create(ValidatorSessionDescription& desc, td::uint32 src_idx, - const ValidatorSessionRootHash& root_hash, const ValidatorSessionFileHash& file_hash, - const ValidatorSessionCollatedDataFileHash& collated_data_file_hash) { - auto hash = create_hash(desc, src_idx, root_hash, file_hash, collated_data_file_hash); - auto r = lookup(desc, src_idx, root_hash, file_hash, collated_data_file_hash, hash, true); - if (r) { - return r; - } - - auto candidate_id = desc.candidate_id(src_idx, root_hash, file_hash, collated_data_file_hash); - - return new (desc, true) SentBlock{desc, src_idx, root_hash, file_hash, collated_data_file_hash, candidate_id, hash}; - } - static const SentBlock* create(ValidatorSessionDescription& desc, const ValidatorSessionCandidateId& zero) { - CHECK(zero.is_zero()); - auto hash = create_hash(desc, 0, ValidatorSessionRootHash::zero(), ValidatorSessionFileHash::zero(), - ValidatorSessionCollatedDataFileHash::zero()); - - return new (desc, true) SentBlock{desc, - 0, - ValidatorSessionRootHash::zero(), - ValidatorSessionFileHash::zero(), - ValidatorSessionCollatedDataFileHash::zero(), - zero, - hash}; - } - static const SentBlock* move_to_persistent(ValidatorSessionDescription& desc, const SentBlock* b) { - if (desc.is_persistent(b)) { - return b; - } - auto r = lookup(desc, b->src_idx_, b->root_hash_, b->file_hash_, b->collated_data_file_hash_, b->hash_, false); - if (r) { - return r; - } - - return new (desc, false) SentBlock{ - desc, b->src_idx_, b->root_hash_, b->file_hash_, b->collated_data_file_hash_, b->candidate_id_, b->hash_}; - } - SentBlock(ValidatorSessionDescription& desc, td::uint32 src_idx, ValidatorSessionRootHash root_hash, - ValidatorSessionFileHash file_hash, ValidatorSessionCollatedDataFileHash collated_data_file_hash, - ValidatorSessionCandidateId candidate_id, HashType hash) - : RootObject(sizeof(SentBlock)) - , src_idx_(src_idx) - , root_hash_(std::move(root_hash)) - , file_hash_(std::move(file_hash)) - , collated_data_file_hash_(std::move(collated_data_file_hash)) - , candidate_id_(candidate_id) - , hash_(std::move(hash)) { - desc.update_hash(this, hash_); - } - auto get_src_idx() const { - return src_idx_; - } - auto get_root_hash() const { - return root_hash_; - } - auto get_file_hash() const { - return file_hash_; - } - auto get_collated_data_file_hash() const { - return collated_data_file_hash_; - } - static ValidatorSessionCandidateId get_block_id(const SentBlock* block) { - return block ? block->candidate_id_ : skip_round_candidate_id(); - } - HashType get_hash(ValidatorSessionDescription& desc) const { - return hash_; - } - bool operator<(const SentBlock& block) const { - if (src_idx_ < block.src_idx_) { - return true; - } - if (src_idx_ > block.src_idx_) { - return false; - } - if (candidate_id_ < block.candidate_id_) { - return true; - } - return false; - } - struct Compare { - bool operator()(const SentBlock* a, const SentBlock* b) const { - return *a < *b; - } - }; - - private: - const td::uint32 src_idx_; - const ValidatorSessionRootHash root_hash_; - const ValidatorSessionFileHash file_hash_; - const ValidatorSessionCollatedDataFileHash collated_data_file_hash_; - const ValidatorSessionCandidateId candidate_id_; - const HashType hash_; -}; - -class SessionBlockCandidate : public ValidatorSessionDescription::RootObject { - public: - static HashType create_hash(ValidatorSessionDescription& desc, HashType block, HashType approved) { - auto obj = create_tl_object(block, approved); - return desc.compute_hash(serialize_tl_object(obj, true).as_slice()); - } - static bool compare(const RootObject* r, const SentBlock* block, const SessionBlockCandidateSignatureVector* approved, - HashType hash) { - if (!r || r->get_size() < sizeof(SessionBlockCandidate)) { - return false; - } - auto R = static_cast(r); - return R->block_ == block && R->approved_by_ == approved && R->hash_ == hash; - } - static auto lookup(ValidatorSessionDescription& desc, const SentBlock* block, - const SessionBlockCandidateSignatureVector* approved, HashType hash, bool temp) { - auto r = desc.get_by_hash(hash, temp); - if (compare(r, block, approved, hash)) { - desc.on_reuse(); - return static_cast(r); - } - return static_cast(nullptr); - } - static const SessionBlockCandidate* create(ValidatorSessionDescription& desc, const SentBlock* block, - const SessionBlockCandidateSignatureVector* approved) { - auto hash = create_hash(desc, get_vs_hash(desc, block), get_vs_hash(desc, approved)); - - auto r = lookup(desc, block, approved, hash, true); - if (r) { - return r; - } - - return new (desc, true) SessionBlockCandidate(desc, block, approved, hash); - } - static const SessionBlockCandidate* create(ValidatorSessionDescription& desc, const SentBlock* block) { - std::vector v; - v.resize(desc.get_total_nodes(), nullptr); - auto vec = SessionBlockCandidateSignatureVector::create(desc, std::move(v)); - return create(desc, block, vec); - } - static const SessionBlockCandidate* move_to_persistent(ValidatorSessionDescription& desc, - const SessionBlockCandidate* b) { - if (desc.is_persistent(b)) { - return b; - } - auto block = SentBlock::move_to_persistent(desc, b->block_); - auto approved = SessionBlockCandidateSignatureVector::move_to_persistent(desc, b->approved_by_); - auto r = lookup(desc, block, approved, b->hash_, false); - if (r) { - return r; - } - - return new (desc, false) SessionBlockCandidate{desc, block, approved, b->hash_}; - } - SessionBlockCandidate(ValidatorSessionDescription& desc, const SentBlock* block, - const SessionBlockCandidateSignatureVector* approved, HashType hash) - : RootObject{sizeof(SessionBlockCandidate)}, block_(block), approved_by_(approved), hash_(hash) { - desc.update_hash(this, hash_); - } - static const SessionBlockCandidate* merge(ValidatorSessionDescription& desc, const SessionBlockCandidate* l, - const SessionBlockCandidate* r); - auto get_block() const { - return block_; - } - auto get_id() const { - return SentBlock::get_block_id(block_); - } - auto get_src_idx() const { - return block_ ? block_->get_src_idx() : std::numeric_limits::max(); - } - bool check_block_is_approved_by(td::uint32 src_idx) const { - return approved_by_->at(src_idx); - } - bool check_block_is_approved(ValidatorSessionDescription& desc) const; - auto get_hash(ValidatorSessionDescription& desc) const { - return hash_; - } - auto get_approvers_list() const { - return approved_by_; - } - static const SessionBlockCandidate* push(ValidatorSessionDescription& desc, const SessionBlockCandidate* state, - td::uint32 src_idx, const SessionBlockCandidateSignature* sig) { - CHECK(state); - if (state->approved_by_->at(src_idx)) { - return state; - } - return create(desc, state->block_, - SessionBlockCandidateSignatureVector::change(desc, state->approved_by_, src_idx, sig)); - } - class Compare { - public: - bool operator()(const SessionBlockCandidate* l, const SessionBlockCandidate* r) { - return l->get_id() < r->get_id(); - } - }; - - private: - const SentBlock* block_; - const SessionBlockCandidateSignatureVector* approved_by_; - const HashType hash_; -}; - -class SessionVoteCandidate : public ValidatorSessionDescription::RootObject { - public: - static HashType create_hash(ValidatorSessionDescription& desc, HashType block, HashType voted) { - auto obj = create_tl_object(block, voted); - return desc.compute_hash(serialize_tl_object(obj, true).as_slice()); - } - static bool compare(const RootObject* r, const SentBlock* block, const CntVector* voted, HashType hash) { - if (!r || r->get_size() < sizeof(SessionVoteCandidate)) { - return false; - } - auto R = static_cast(r); - return R->block_ == block && R->voted_by_ == voted && R->hash_ == hash; - } - static auto lookup(ValidatorSessionDescription& desc, const SentBlock* block, const CntVector* voted, - HashType hash, bool temp) { - auto r = desc.get_by_hash(hash, temp); - if (compare(r, block, voted, hash)) { - desc.on_reuse(); - return static_cast(r); - } - return static_cast(nullptr); - } - static const SessionVoteCandidate* create(ValidatorSessionDescription& desc, const SentBlock* block, - const CntVector* voted) { - auto hash = create_hash(desc, get_vs_hash(desc, block), get_vs_hash(desc, voted)); - - auto r = lookup(desc, block, voted, hash, true); - if (r) { - return r; - } - - return new (desc, true) SessionVoteCandidate(desc, block, voted, hash); - } - static const SessionVoteCandidate* create(ValidatorSessionDescription& desc, const SentBlock* block) { - std::vector v; - v.resize(desc.get_total_nodes(), false); - auto vec = CntVector::create(desc, std::move(v)); - return create(desc, block, vec); - } - static const SessionVoteCandidate* move_to_persistent(ValidatorSessionDescription& desc, - const SessionVoteCandidate* b) { - if (desc.is_persistent(b)) { - return b; - } - auto block = SentBlock::move_to_persistent(desc, b->block_); - auto voted = CntVector::move_to_persistent(desc, b->voted_by_); - auto r = lookup(desc, block, voted, b->hash_, false); - if (r) { - return r; - } - - return new (desc, false) SessionVoteCandidate{desc, block, voted, b->hash_}; - } - SessionVoteCandidate(ValidatorSessionDescription& desc, const SentBlock* block, const CntVector* voted, - HashType hash) - : RootObject{sizeof(SessionVoteCandidate)}, block_(block), voted_by_(voted), hash_(hash) { - desc.update_hash(this, hash_); - } - static const SessionVoteCandidate* merge(ValidatorSessionDescription& desc, const SessionVoteCandidate* l, - const SessionVoteCandidate* r); - auto get_block() const { - return block_; - } - auto get_id() const { - return SentBlock::get_block_id(block_); - } - auto get_src_idx() const { - return block_ ? block_->get_src_idx() : std::numeric_limits::max(); - } - bool check_block_is_voted_by(td::uint32 src_idx) const { - return voted_by_->at(src_idx); - } - bool check_block_is_voted(ValidatorSessionDescription& desc) const; - auto get_hash(ValidatorSessionDescription& desc) const { - return hash_; - } - auto get_voters_list() const { - return voted_by_; - } - static const SessionVoteCandidate* push(ValidatorSessionDescription& desc, const SessionVoteCandidate* state, - td::uint32 src_idx) { - CHECK(state); - if (state->voted_by_->at(src_idx)) { - return state; - } - return create(desc, state->block_, CntVector::change(desc, state->voted_by_, src_idx, true)); - } - class Compare { - public: - bool operator()(const SessionVoteCandidate* l, const SessionVoteCandidate* r) { - return l->get_id() < r->get_id(); - } - }; - - private: - const SentBlock* block_; - const CntVector* voted_by_; - const HashType hash_; -}; - -//using SentBlockVector = CntSortedVector; - -class ValidatorSessionRoundState; class ValidatorSessionOldRoundState : public ValidatorSessionDescription::RootObject { public: static HashType create_hash(ValidatorSessionDescription& desc, td::uint32 seqno, HashType block, HashType signatures, @@ -582,188 +178,6 @@ class ValidatorSessionOldRoundState : public ValidatorSessionDescription::RootOb const HashType hash_; }; -using VoteVector = CntSortedVector; -using ApproveVector = CntSortedVector; - -class ValidatorSessionRoundAttemptState : public ValidatorSessionDescription::RootObject { - public: - static HashType create_hash(ValidatorSessionDescription& desc, td::uint32 seqno, HashType votes, - HashType precommitted, bool vote_for_inited, HashType vote_for) { - auto obj = create_tl_object(seqno, votes, precommitted, - vote_for_inited, vote_for); - return desc.compute_hash(serialize_tl_object(obj, true).as_slice()); - } - static bool compare(const RootObject* r, td::uint32 seqno, const VoteVector* votes, - const CntVector* precommitted, const SentBlock* vote_for, bool vote_for_inited, - HashType hash) { - if (!r || r->get_size() < sizeof(ValidatorSessionRoundAttemptState)) { - return false; - } - auto R = static_cast(r); - return R->seqno_ == seqno && R->votes_ == votes && R->precommitted_ == precommitted && R->vote_for_ == vote_for && - R->vote_for_inited_ == vote_for_inited && R->hash_ == hash; - } - static auto lookup(ValidatorSessionDescription& desc, td::uint32 seqno, const VoteVector* votes, - const CntVector* precommitted, const SentBlock* vote_for, bool vote_for_inited, - HashType hash, bool temp) { - auto r = desc.get_by_hash(hash, temp); - if (compare(r, seqno, votes, precommitted, vote_for, vote_for_inited, hash)) { - desc.on_reuse(); - return static_cast(r); - } - return static_cast(nullptr); - } - static const ValidatorSessionRoundAttemptState* move_to_persistent(ValidatorSessionDescription& desc, - const ValidatorSessionRoundAttemptState* b) { - if (desc.is_persistent(b)) { - return b; - } - auto votes = VoteVector::move_to_persistent(desc, b->votes_); - auto precommitted = CntVector::move_to_persistent(desc, b->precommitted_); - auto vote_for = SentBlock::move_to_persistent(desc, b->vote_for_); - - auto r = lookup(desc, b->seqno_, votes, precommitted, vote_for, b->vote_for_inited_, b->hash_, false); - if (r) { - return r; - } - - return new (desc, false) ValidatorSessionRoundAttemptState{desc, b->seqno_, votes, precommitted, - vote_for, b->vote_for_inited_, b->hash_}; - } - - static const ValidatorSessionRoundAttemptState* create(ValidatorSessionDescription& desc, td::uint32 seqno, - const VoteVector* votes, const CntVector* precommitted, - const SentBlock* vote_for, bool vote_for_inited) { - auto hash = create_hash(desc, seqno, get_vs_hash(desc, votes), get_vs_hash(desc, precommitted), - get_vs_hash(desc, vote_for), vote_for_inited); - - auto r = lookup(desc, seqno, votes, precommitted, vote_for, vote_for_inited, hash, true); - if (r) { - return r; - } - - return new (desc, true) - ValidatorSessionRoundAttemptState(desc, seqno, votes, precommitted, vote_for, vote_for_inited, hash); - } - static const ValidatorSessionRoundAttemptState* create(ValidatorSessionDescription& desc, td::uint32 seqno) { - std::vector x; - x.resize(desc.get_total_nodes(), false); - auto p = CntVector::create(desc, std::move(x)); - - return create(desc, seqno, nullptr, p, nullptr, false); - } - - ValidatorSessionRoundAttemptState(ValidatorSessionDescription& desc, td::uint32 seqno, const VoteVector* votes, - const CntVector* precommitted, const SentBlock* vote_for, - bool vote_for_inited, HashType hash) - : RootObject{sizeof(ValidatorSessionRoundAttemptState)} - , seqno_(seqno) - , votes_(votes) - , precommitted_(precommitted) - , vote_for_(vote_for) - , vote_for_inited_(vote_for_inited) - , hash_(std::move(hash)) { - desc.update_hash(this, hash_); - } - auto get_hash(ValidatorSessionDescription& desc) const { - return hash_; - } - auto get_seqno() const { - return seqno_; - } - auto get_votes() const { - return votes_; - } - auto get_precommits() const { - return precommitted_; - } - const SentBlock* get_voted_block(ValidatorSessionDescription& desc, bool& f) const; - const SentBlock* get_vote_for_block(ValidatorSessionDescription& desc, bool& f) const { - f = vote_for_inited_; - return vote_for_; - } - bool check_attempt_is_precommitted(ValidatorSessionDescription& desc) const; - bool check_vote_received_from(td::uint32 src_idx) const; - bool check_precommit_received_from(td::uint32 src_idx) const; - - bool operator<(const ValidatorSessionRoundAttemptState& right) const { - return seqno_ < right.seqno_; - } - struct Compare { - bool operator()(const ValidatorSessionRoundAttemptState* a, const ValidatorSessionRoundAttemptState* b) const { - return *a < *b; - } - }; - - static const ValidatorSessionRoundAttemptState* merge(ValidatorSessionDescription& desc, - const ValidatorSessionRoundAttemptState* left, - const ValidatorSessionRoundAttemptState* right); - static const ValidatorSessionRoundAttemptState* action(ValidatorSessionDescription& desc, - const ValidatorSessionRoundAttemptState* state, - td::uint32 src_idx, td::uint32 att, - const ton_api::validatorSession_message_voteFor& act, - const ValidatorSessionRoundState* round); - static const ValidatorSessionRoundAttemptState* action(ValidatorSessionDescription& desc, - const ValidatorSessionRoundAttemptState* state, - td::uint32 src_idx, td::uint32 att, - const ton_api::validatorSession_message_vote& act, - const ValidatorSessionRoundState* round); - static const ValidatorSessionRoundAttemptState* action(ValidatorSessionDescription& desc, - const ValidatorSessionRoundAttemptState* state, - td::uint32 src_idx, td::uint32 att, - const ton_api::validatorSession_message_precommit& act, - const ValidatorSessionRoundState* round); - static const ValidatorSessionRoundAttemptState* action(ValidatorSessionDescription& desc, - const ValidatorSessionRoundAttemptState* state, - td::uint32 src_idx, td::uint32 att, - const ton_api::validatorSession_message_empty& act, - const ValidatorSessionRoundState* round); - static const ValidatorSessionRoundAttemptState* action(ValidatorSessionDescription& desc, - const ValidatorSessionRoundAttemptState* state, - td::uint32 src_idx, td::uint32 att, - const ton_api::validatorSession_round_Message* action, - const ValidatorSessionRoundState* round); - template - static const ValidatorSessionRoundAttemptState* action(ValidatorSessionDescription& desc, - const ValidatorSessionRoundAttemptState* state, - td::uint32 src_idx, td::uint32 att, const T& action, - const ValidatorSessionRoundState* round) { - UNREACHABLE(); - } - static const ValidatorSessionRoundAttemptState* try_vote(ValidatorSessionDescription& desc, - const ValidatorSessionRoundAttemptState* state, - td::uint32 src_idx, td::uint32 att, - const ValidatorSessionRoundState* round, - const ton_api::validatorSession_round_Message* cmp, - bool& made); - static const ValidatorSessionRoundAttemptState* try_precommit(ValidatorSessionDescription& desc, - const ValidatorSessionRoundAttemptState* state, - td::uint32 src_idx, td::uint32 att, - const ValidatorSessionRoundState* round, - const ton_api::validatorSession_round_Message* cmp, - bool& made); - static const ValidatorSessionRoundAttemptState* make_one(ValidatorSessionDescription& desc, - const ValidatorSessionRoundAttemptState* state, - td::uint32 src_idx, td::uint32 att, - const ValidatorSessionRoundState* round, - const ton_api::validatorSession_round_Message* cmp, - bool& made); - tl_object_ptr create_action(ValidatorSessionDescription& desc, - const ValidatorSessionRoundState* round, - td::uint32 src_idx, td::uint32 att) const; - void dump(ValidatorSessionDescription& desc, td::StringBuilder& sb) const; - - private: - const td::uint32 seqno_; - const VoteVector* votes_; - const CntVector* precommitted_; - const SentBlock* vote_for_; - const bool vote_for_inited_; - const HashType hash_; -}; - -using AttemptVector = - CntSortedVector; class ValidatorSessionRoundState : public ValidatorSessionDescription::RootObject { public: diff --git a/validator-session/validator-session-types.h b/validator-session/validator-session-types.h index 357f6217b..579574786 100644 --- a/validator-session/validator-session-types.h +++ b/validator-session/validator-session-types.h @@ -27,6 +27,12 @@ namespace ton { namespace validatorsession { +constexpr int VERBOSITY_NAME(VALIDATOR_SESSION_WARNING) = verbosity_WARNING; +constexpr int VERBOSITY_NAME(VALIDATOR_SESSION_NOTICE) = verbosity_DEBUG; +constexpr int VERBOSITY_NAME(VALIDATOR_SESSION_INFO) = verbosity_DEBUG; +constexpr int VERBOSITY_NAME(VALIDATOR_SESSION_DEBUG) = verbosity_DEBUG; +constexpr int VERBOSITY_NAME(VALIDATOR_SESSION_EXTRA_DEBUG) = verbosity_DEBUG + 1; + using ValidatorSessionRootHash = td::Bits256; using ValidatorSessionFileHash = td::Bits256; using ValidatorSessionCollatedDataFileHash = td::Bits256; diff --git a/validator-session/validator-session.cpp b/validator-session/validator-session.cpp index 2902b082c..e08d8a7e0 100644 --- a/validator-session/validator-session.cpp +++ b/validator-session/validator-session.cpp @@ -203,7 +203,23 @@ void ValidatorSessionImpl::preprocess_block(catchain::CatChainBlock *block) { << "ms: state=" << state->get_hash(description()); } -void ValidatorSessionImpl::process_broadcast(PublicKeyHash src, td::BufferSlice data) { +bool ValidatorSessionImpl::ensure_candidate_unique(td::uint32 src_idx, td::uint32 round, + ValidatorSessionCandidateId block_id) { + auto it = src_round_candidate_[src_idx].find(round); + if (it != src_round_candidate_[src_idx].end() && it->second != block_id) { + VLOG(VALIDATOR_SESSION_WARNING) << this << "[node " << description_->get_source_adnl_id(src_idx) << "][candidate " + << block_id << "]: this node already has candidate in round " << round; + return false; + } + src_round_candidate_[src_idx][round] = block_id; + return true; +} + +void ValidatorSessionImpl::process_broadcast(PublicKeyHash src, td::BufferSlice data, + td::optional expected_id, + bool is_overlay_broadcast) { + // Note: src is not necessarily equal to the sender of this message: + // If requested using get_broadcast_p2p, src is the creator of the block, sender possibly is some other node. auto src_idx = description().get_source_idx(src); auto R = fetch_tl_object(data.clone(), true); if (R.is_error()) { @@ -230,6 +246,12 @@ void ValidatorSessionImpl::process_broadcast(PublicKeyHash src, td::BufferSlice auto block_round = static_cast(candidate->round_); auto block_id = description().candidate_id(src_idx, candidate->root_hash_, file_hash, collated_data_file_hash); + if (expected_id && expected_id.value() != block_id) { + VLOG(VALIDATOR_SESSION_WARNING) << this << "[node " << src << "][broadcast " << sha256_bits256(data.as_slice()) + << "]: id mismatch"; + return; + } + if ((td::int32)block_round < (td::int32)cur_round_ - MAX_PAST_ROUND_BLOCK || block_round >= cur_round_ + MAX_FUTURE_ROUND_BLOCK) { VLOG(VALIDATOR_SESSION_NOTICE) << this << "[node " << src << "][broadcast " << block_id @@ -250,6 +272,10 @@ void ValidatorSessionImpl::process_broadcast(PublicKeyHash src, td::BufferSlice return; } + if (is_overlay_broadcast && !ensure_candidate_unique(src_idx, block_round, block_id)) { + return; + } + blocks_[block_id] = std::move(candidate); VLOG(VALIDATOR_SESSION_WARNING) << this << ": received broadcast " << block_id; @@ -508,7 +534,11 @@ void ValidatorSessionImpl::try_approve_block(const SentBlock *block) { } if (block) { - auto T = td::Timestamp::at(round_started_at_.at() + description().get_delay(block->get_src_idx()) + 2.0); + if (!ensure_candidate_unique(block->get_src_idx(), cur_round_, SentBlock::get_block_id(block))) { + return; + } + auto T = td::Timestamp::at(round_started_at_.at() + description().get_delay(block->get_src_idx()) + + REQUEST_BROADCAST_P2P_DELAY); auto it = blocks_.find(block_id); if (it != blocks_.end()) { @@ -545,16 +575,18 @@ void ValidatorSessionImpl::try_approve_block(const SentBlock *block) { auto id = description().get_source_id(v[td::Random::fast(0, static_cast(v.size() - 1))]); auto src_id = description().get_source_id(block->get_src_idx()); active_requests_.insert(block_id); - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), id, src_id, print_id = print_id(), - hash = block_id, round = cur_round_](td::Result R) { - td::actor::send_closure(SelfId, &ValidatorSessionImpl::end_request, round, hash); - if (R.is_error()) { - VLOG(VALIDATOR_SESSION_WARNING) - << print_id << ": failed to get candidate " << hash << " from " << id << ": " << R.move_as_error(); - } else { - td::actor::send_closure(SelfId, &ValidatorSessionImpl::process_broadcast, src_id, R.move_as_ok()); - } - }); + auto P = td::PromiseCreator::lambda( + [SelfId = actor_id(this), id, src_id, print_id = print_id(), hash = block_id, round = cur_round_, + candidate_id = SentBlock::get_block_id(block)](td::Result R) { + td::actor::send_closure(SelfId, &ValidatorSessionImpl::end_request, round, hash); + if (R.is_error()) { + VLOG(VALIDATOR_SESSION_WARNING) << print_id << ": failed to get candidate " << hash << " from " << id + << ": " << R.move_as_error(); + } else { + td::actor::send_closure(SelfId, &ValidatorSessionImpl::process_broadcast, src_id, R.move_as_ok(), + candidate_id, false); + } + }); get_broadcast_p2p(id, block->get_file_hash(), block->get_collated_data_file_hash(), description().get_source_id(block->get_src_idx()), cur_round_, block->get_root_hash(), @@ -586,10 +618,11 @@ void ValidatorSessionImpl::get_broadcast_p2p(PublicKeyHash node, ValidatorSessio round, create_tl_object(src.tl(), root_hash, file_hash, collated_data_file_hash)); - td::actor::send_closure(catchain_, &catchain::CatChain::send_query_via, node, "download candidate", - std::move(promise), timeout, serialize_tl_object(obj, true), - description().opts().max_block_size + description().opts().max_collated_data_size + 1024, - rldp_); + td::actor::send_closure( + catchain_, &catchain::CatChain::send_query_via, node, "download candidate", std::move(promise), timeout, + serialize_tl_object(obj, true), + description().opts().max_block_size + description().opts().max_collated_data_size + MAX_CANDIDATE_EXTRA_SIZE, + rldp_); } void ValidatorSessionImpl::check_sign_slot() { @@ -745,7 +778,6 @@ void ValidatorSessionImpl::on_new_round(td::uint32 round) { while (cur_round_ < round) { auto block = real_state_->get_committed_block(description(), cur_round_); - //CHECK(block); auto sigs = real_state_->get_committed_block_signatures(description(), cur_round_); CHECK(sigs); auto approve_sigs = real_state_->get_committed_block_approve_signatures(description(), cur_round_); @@ -838,7 +870,8 @@ void ValidatorSessionImpl::on_catchain_started() { auto broadcast = create_tl_object( src.tl(), round, root_hash, std::move(B.data), std::move(B.collated_data)); td::actor::send_closure(SelfId, &ValidatorSessionImpl::process_broadcast, src, - serialize_tl_object(broadcast, true)); + serialize_tl_object(broadcast, true), td::optional(), + false); } }); callback_->get_approved_candidate(description().get_source_public_key(x->get_src_idx()), x->get_root_hash(), @@ -866,6 +899,7 @@ ValidatorSessionImpl::ValidatorSessionImpl(catchain::CatChainSessionId session_i , overlay_manager_(overlays) , allow_unsafe_self_blocks_resync_(allow_unsafe_self_blocks_resync) { description_ = ValidatorSessionDescription::create(std::move(opts), nodes, local_id); + src_round_candidate_.resize(description_->get_total_nodes()); } void ValidatorSessionImpl::start() { diff --git a/validator-session/validator-session.hpp b/validator-session/validator-session.hpp index e82745547..1717c99f0 100644 --- a/validator-session/validator-session.hpp +++ b/validator-session/validator-session.hpp @@ -74,6 +74,8 @@ class ValidatorSessionImpl : public ValidatorSession { td::BufferSlice signature_; std::map> blocks_; + // src_round_candidate_[src_id][round] -> candidate id + std::vector> src_round_candidate_; catchain::CatChainSessionId unique_hash_; @@ -110,7 +112,8 @@ class ValidatorSessionImpl : public ValidatorSession { td::actor::send_closure(id_, &ValidatorSessionImpl::preprocess_block, block); } void process_broadcast(const PublicKeyHash &src, td::BufferSlice data) override { - td::actor::send_closure(id_, &ValidatorSessionImpl::process_broadcast, src, std::move(data)); + td::actor::send_closure(id_, &ValidatorSessionImpl::process_broadcast, src, std::move(data), + td::optional(), true); } void process_message(const PublicKeyHash &src, td::BufferSlice data) override { td::actor::send_closure(id_, &ValidatorSessionImpl::process_message, src, std::move(data)); @@ -174,7 +177,9 @@ class ValidatorSessionImpl : public ValidatorSession { void process_blocks(std::vector blocks); void finished_processing(); void preprocess_block(catchain::CatChainBlock *block); - void process_broadcast(PublicKeyHash src, td::BufferSlice data); + bool ensure_candidate_unique(td::uint32 src_idx, td::uint32 round, ValidatorSessionCandidateId block_id); + void process_broadcast(PublicKeyHash src, td::BufferSlice data, td::optional expected_id, + bool is_overlay_broadcast); void process_message(PublicKeyHash src, td::BufferSlice data); void process_query(PublicKeyHash src, td::BufferSlice data, td::Promise promise); @@ -206,6 +211,8 @@ class ValidatorSessionImpl : public ValidatorSession { static const size_t MAX_REJECT_REASON_SIZE = 1024; static const td::int32 MAX_FUTURE_ROUND_BLOCK = 100; static const td::int32 MAX_PAST_ROUND_BLOCK = 20; + constexpr static const double REQUEST_BROADCAST_P2P_DELAY = 2.0; + static const td::uint32 MAX_CANDIDATE_EXTRA_SIZE = 1024; }; } // namespace validatorsession diff --git a/validator/db/archive-manager.cpp b/validator/db/archive-manager.cpp index bd239bfc4..91b7b880b 100644 --- a/validator/db/archive-manager.cpp +++ b/validator/db/archive-manager.cpp @@ -74,7 +74,7 @@ void ArchiveManager::add_handle(BlockHandle handle, td::Promise promis } void ArchiveManager::update_handle(BlockHandle handle, td::Promise promise) { - FileDescription *f; + const FileDescription *f; if (handle->handle_moved_to_archive()) { CHECK(handle->inited_unix_time()); if (!handle->need_flush()) { @@ -439,15 +439,15 @@ void ArchiveManager::check_persistent_state(BlockIdExt block_id, BlockIdExt mast void ArchiveManager::get_block_by_unix_time(AccountIdPrefixFull account_id, UnixTime ts, td::Promise promise) { - auto f = get_file_desc_by_unix_time(account_id, ts, false); - if (f) { - auto n = f; - do { - n = get_next_file_desc(n); - } while (n != nullptr && !n->has_account_prefix(account_id)); + auto f1 = get_file_desc_by_unix_time(account_id, ts, false); + auto f2 = get_next_file_desc(f1, account_id, false); + if (!f1) { + std::swap(f1, f2); + } + if (f1) { td::actor::ActorId aid; - if (n) { - aid = n->file_actor_id(); + if (f2) { + aid = f2->file_actor_id(); } auto P = td::PromiseCreator::lambda( [aid, account_id, ts, promise = std::move(promise)](td::Result R) mutable { @@ -457,7 +457,7 @@ void ArchiveManager::get_block_by_unix_time(AccountIdPrefixFull account_id, Unix td::actor::send_closure(aid, &ArchiveSlice::get_block_by_unix_time, account_id, ts, std::move(promise)); } }); - td::actor::send_closure(f->file_actor_id(), &ArchiveSlice::get_block_by_unix_time, account_id, ts, std::move(P)); + td::actor::send_closure(f1->file_actor_id(), &ArchiveSlice::get_block_by_unix_time, account_id, ts, std::move(P)); } else { promise.set_error(td::Status::Error(ErrorCode::notready, "ts not in db")); } @@ -465,15 +465,15 @@ void ArchiveManager::get_block_by_unix_time(AccountIdPrefixFull account_id, Unix void ArchiveManager::get_block_by_lt(AccountIdPrefixFull account_id, LogicalTime lt, td::Promise promise) { - auto f = get_file_desc_by_lt(account_id, lt, false); - if (f) { - auto n = f; - do { - n = get_next_file_desc(n); - } while (n != nullptr && !n->has_account_prefix(account_id)); + auto f1 = get_file_desc_by_lt(account_id, lt, false); + auto f2 = get_next_file_desc(f1, account_id, false); + if (!f1) { + std::swap(f1, f2); + } + if (f1) { td::actor::ActorId aid; - if (n) { - aid = n->file_actor_id(); + if (f2) { + aid = f2->file_actor_id(); } auto P = td::PromiseCreator::lambda( [aid, account_id, lt, promise = std::move(promise)](td::Result R) mutable { @@ -483,7 +483,7 @@ void ArchiveManager::get_block_by_lt(AccountIdPrefixFull account_id, LogicalTime td::actor::send_closure(aid, &ArchiveSlice::get_block_by_lt, account_id, lt, std::move(promise)); } }); - td::actor::send_closure(f->file_actor_id(), &ArchiveSlice::get_block_by_lt, account_id, lt, std::move(P)); + td::actor::send_closure(f1->file_actor_id(), &ArchiveSlice::get_block_by_lt, account_id, lt, std::move(P)); } else { promise.set_error(td::Status::Error(ErrorCode::notready, "lt not in db")); } @@ -558,11 +558,16 @@ void ArchiveManager::deleted_package(PackageId id, td::Promise promise auto it = m.find(id); CHECK(it != m.end()); CHECK(it->second.deleted); - it->second.clear_actor_id(); + it->second.file.reset(); promise.set_value(td::Unit()); } void ArchiveManager::load_package(PackageId id) { + auto &m = get_file_map(id); + if (m.count(id)) { + LOG(WARNING) << "Duplicate id " << id.name(); + return; + } auto key = create_serialize_tl_object(id.id, id.key, id.temp); std::string value; @@ -595,11 +600,11 @@ void ArchiveManager::load_package(PackageId id) { desc.file = td::actor::create_actor("slice", id.id, id.key, id.temp, false, db_root_); - get_file_map(id).emplace(id, std::move(desc)); + m.emplace(id, std::move(desc)); } -ArchiveManager::FileDescription *ArchiveManager::get_file_desc(ShardIdFull shard, PackageId id, BlockSeqno seqno, - UnixTime ts, LogicalTime lt, bool force) { +const ArchiveManager::FileDescription *ArchiveManager::get_file_desc(ShardIdFull shard, PackageId id, BlockSeqno seqno, + UnixTime ts, LogicalTime lt, bool force) { auto &f = get_file_map(id); auto it = f.find(id); if (it != f.end()) { @@ -607,7 +612,7 @@ ArchiveManager::FileDescription *ArchiveManager::get_file_desc(ShardIdFull shard return nullptr; } if (force && !id.temp) { - update_desc(it->second, shard, seqno, ts, lt); + update_desc(f, it->second, shard, seqno, ts, lt); } return &it->second; } @@ -618,17 +623,18 @@ ArchiveManager::FileDescription *ArchiveManager::get_file_desc(ShardIdFull shard return add_file_desc(shard, id, seqno, ts, lt); } -ArchiveManager::FileDescription *ArchiveManager::add_file_desc(ShardIdFull shard, PackageId id, BlockSeqno seqno, - UnixTime ts, LogicalTime lt) { +const ArchiveManager::FileDescription *ArchiveManager::add_file_desc(ShardIdFull shard, PackageId id, BlockSeqno seqno, + UnixTime ts, LogicalTime lt) { auto &f = get_file_map(id); CHECK(f.count(id) == 0); - FileDescription desc{id, false}; + FileDescription new_desc{id, false}; td::mkdir(db_root_ + id.path()).ensure(); std::string prefix = PSTRING() << db_root_ << id.path() << id.name(); - desc.file = td::actor::create_actor("slice", id.id, id.key, id.temp, false, db_root_); + new_desc.file = td::actor::create_actor("slice", id.id, id.key, id.temp, false, db_root_); + const FileDescription &desc = f.emplace(id, std::move(new_desc)); if (!id.temp) { - update_desc(desc, shard, seqno, ts, lt); + update_desc(f, desc, shard, seqno, ts, lt); } std::vector> vec; @@ -652,7 +658,6 @@ ArchiveManager::FileDescription *ArchiveManager::add_file_desc(ShardIdFull shard for (auto &e : temp_files_) { tt.push_back(e.first.id); } - (id.temp ? tt : (id.key ? tk : t)).push_back(id.id); index_ ->set(create_serialize_tl_object().as_slice(), create_serialize_tl_object(std::move(t), std::move(tk), std::move(tt)) @@ -668,17 +673,16 @@ ArchiveManager::FileDescription *ArchiveManager::add_file_desc(ShardIdFull shard .ensure(); } index_->commit_transaction().ensure(); - - return &f.emplace(id, std::move(desc)).first->second; + return &desc; } -void ArchiveManager::update_desc(FileDescription &desc, ShardIdFull shard, BlockSeqno seqno, UnixTime ts, - LogicalTime lt) { +void ArchiveManager::update_desc(FileMap &f, const FileDescription &desc, ShardIdFull shard, BlockSeqno seqno, + UnixTime ts, LogicalTime lt) { auto it = desc.first_blocks.find(shard); if (it != desc.first_blocks.end() && it->second.seqno <= seqno) { return; } - desc.first_blocks[shard] = FileDescription::Desc{seqno, ts, lt}; + f.set_shard_first_block(desc, shard, FileDescription::Desc{seqno, ts, lt}); std::vector> vec; for (auto &e : desc.first_blocks) { vec.push_back(create_tl_object(e.first.workchain, e.first.shard, @@ -694,150 +698,91 @@ void ArchiveManager::update_desc(FileDescription &desc, ShardIdFull shard, Block index_->commit_transaction().ensure(); } -ArchiveManager::FileDescription *ArchiveManager::get_file_desc_by_seqno(ShardIdFull shard, BlockSeqno seqno, - bool key_block) { - auto &f = get_file_map(PackageId{0, key_block, false}); - for (auto it = f.rbegin(); it != f.rend(); it++) { - auto i = it->second.first_blocks.find(shard); - if (i != it->second.first_blocks.end() && i->second.seqno <= seqno) { - if (it->second.deleted) { - return nullptr; - } else { - return &it->second; - } - } - } - return nullptr; +const ArchiveManager::FileDescription *ArchiveManager::get_file_desc_by_seqno(ShardIdFull shard, BlockSeqno seqno, + bool key_block) { + return get_file_map(PackageId{0, key_block, false}).get_file_desc_by_seqno(shard, seqno); } -ArchiveManager::FileDescription *ArchiveManager::get_file_desc_by_unix_time(ShardIdFull shard, UnixTime ts, - bool key_block) { - auto &f = get_file_map(PackageId{0, key_block, false}); - for (auto it = f.rbegin(); it != f.rend(); it++) { - auto i = it->second.first_blocks.find(shard); - if (i != it->second.first_blocks.end() && i->second.ts <= ts) { - if (it->second.deleted) { - return nullptr; - } else { - return &it->second; - } - } - } - return nullptr; +const ArchiveManager::FileDescription *ArchiveManager::get_file_desc_by_unix_time(ShardIdFull shard, UnixTime ts, + bool key_block) { + return get_file_map(PackageId{0, key_block, false}).get_file_desc_by_unix_time(shard, ts); } -ArchiveManager::FileDescription *ArchiveManager::get_file_desc_by_lt(ShardIdFull shard, LogicalTime lt, - bool key_block) { - auto &f = get_file_map(PackageId{0, key_block, false}); - for (auto it = f.rbegin(); it != f.rend(); it++) { - auto i = it->second.first_blocks.find(shard); - if (i != it->second.first_blocks.end() && i->second.lt <= lt) { - if (it->second.deleted) { - return nullptr; - } else { - return &it->second; - } - } - } - return nullptr; +const ArchiveManager::FileDescription *ArchiveManager::get_file_desc_by_lt(ShardIdFull shard, LogicalTime lt, + bool key_block) { + return get_file_map(PackageId{0, key_block, false}).get_file_desc_by_lt(shard, lt); } -ArchiveManager::FileDescription *ArchiveManager::get_file_desc_by_seqno(AccountIdPrefixFull account, BlockSeqno seqno, - bool key_block) { - auto &f = get_file_map(PackageId{0, key_block, false}); +const ArchiveManager::FileDescription *ArchiveManager::get_file_desc_by_seqno(AccountIdPrefixFull account, + BlockSeqno seqno, bool key_block) { if (account.is_masterchain()) { return get_file_desc_by_seqno(ShardIdFull{masterchainId}, seqno, key_block); } - for (auto it = f.rbegin(); it != f.rend(); it++) { - if (it->second.deleted) { - continue; - } - bool found = false; - for (int i = 0; i < 60; i++) { - auto shard = shard_prefix(account, i); - auto it2 = it->second.first_blocks.find(shard); - if (it2 != it->second.first_blocks.end()) { - if (it2->second.seqno <= seqno) { - return &it->second; - } - found = true; - } else if (found) { - break; - } + auto &f = get_file_map(PackageId{0, key_block, false}); + const FileDescription *result = nullptr; + for (int i = 0; i <= 60; i++) { + const FileDescription *desc = f.get_file_desc_by_seqno(shard_prefix(account, i), seqno); + if (desc && (!result || result->id < desc->id)) { + result = desc; + } else if (result && (!desc || desc->id < result->id)) { + break; } } - return nullptr; + return result; } -ArchiveManager::FileDescription *ArchiveManager::get_file_desc_by_unix_time(AccountIdPrefixFull account, UnixTime ts, - bool key_block) { - auto &f = get_file_map(PackageId{0, key_block, false}); +const ArchiveManager::FileDescription *ArchiveManager::get_file_desc_by_unix_time(AccountIdPrefixFull account, + UnixTime ts, bool key_block) { if (account.is_masterchain()) { return get_file_desc_by_unix_time(ShardIdFull{masterchainId}, ts, key_block); } - for (auto it = f.rbegin(); it != f.rend(); it++) { - if (it->second.deleted) { - continue; - } - bool found = false; - for (int i = 0; i < 60; i++) { - auto shard = shard_prefix(account, i); - auto it2 = it->second.first_blocks.find(shard); - if (it2 != it->second.first_blocks.end()) { - if (it2->second.ts <= ts) { - return &it->second; - } - found = true; - } else if (found) { - break; - } + auto &f = get_file_map(PackageId{0, key_block, false}); + const FileDescription *result = nullptr; + for (int i = 0; i <= 60; i++) { + const FileDescription *desc = f.get_file_desc_by_unix_time(shard_prefix(account, i), ts); + if (desc && (!result || result->id < desc->id)) { + result = desc; + } else if (result && (!desc || desc->id < result->id)) { + break; } } - return nullptr; + return result; } -ArchiveManager::FileDescription *ArchiveManager::get_file_desc_by_lt(AccountIdPrefixFull account, LogicalTime lt, - bool key_block) { - auto &f = get_file_map(PackageId{0, key_block, false}); +const ArchiveManager::FileDescription *ArchiveManager::get_file_desc_by_lt(AccountIdPrefixFull account, LogicalTime lt, + bool key_block) { if (account.is_masterchain()) { return get_file_desc_by_lt(ShardIdFull{masterchainId}, lt, key_block); } - for (auto it = f.rbegin(); it != f.rend(); it++) { - if (it->second.deleted) { - continue; - } - bool found = false; - for (int i = 0; i < 60; i++) { - auto shard = shard_prefix(account, i); - auto it2 = it->second.first_blocks.find(shard); - if (it2 != it->second.first_blocks.end()) { - if (it2->second.lt <= lt) { - return &it->second; - } - found = true; - } else if (found) { - break; - } + auto &f = get_file_map(PackageId{0, key_block, false}); + const FileDescription *result = nullptr; + for (int i = 0; i <= 60; i++) { + const FileDescription *desc = f.get_file_desc_by_lt(shard_prefix(account, i), lt); + if (desc && (!result || result->id < desc->id)) { + result = desc; + } else if (result && (!desc || desc->id < result->id)) { + break; } } - return nullptr; + return result; } -ArchiveManager::FileDescription *ArchiveManager::get_next_file_desc(FileDescription *f) { - auto &m = get_file_map(f->id); - auto it = m.find(f->id); - CHECK(it != m.end()); - while (true) { - it++; - if (it == m.end()) { - return nullptr; - } else if (!it->second.deleted) { - return &it->second; +const ArchiveManager::FileDescription *ArchiveManager::get_next_file_desc(const FileDescription *f, + AccountIdPrefixFull shard, bool key_block) { + auto &m = get_file_map(PackageId{0, key_block, false}); + const FileDescription *result = nullptr; + for (int i = 0; i <= 60; i++) { + const FileDescription *desc = m.get_next_file_desc(shard_prefix(shard, i), f); + if (desc && (!result || desc->id < result->id)) { + result = desc; + } else if (result && (!desc || result->id < desc->id)) { + break; } } + return result; } -ArchiveManager::FileDescription *ArchiveManager::get_temp_file_desc_by_idx(PackageId idx) { +const ArchiveManager::FileDescription *ArchiveManager::get_temp_file_desc_by_idx(PackageId idx) { auto it = temp_files_.find(idx); if (it != temp_files_.end()) { if (it->second.deleted) { @@ -1257,14 +1202,100 @@ void ArchiveManager::truncate(BlockSeqno masterchain_seqno, ConstBlockHandle han } } -bool ArchiveManager::FileDescription::has_account_prefix(AccountIdPrefixFull account_id) const { - for (int i = 0; i < 60; i++) { - auto shard = shard_prefix(account_id, i); - if (first_blocks.count(shard)) { - return true; - } +void ArchiveManager::FileMap::shard_index_add(const FileDescription &desc) { + for (const auto &p : desc.first_blocks) { + ShardIndex &s = shards_[p.first]; + s.seqno_index_[p.second.seqno] = &desc; + s.lt_index_[p.second.lt] = &desc; + s.unix_time_index_[p.second.ts] = &desc; + s.packages_index_[desc.id] = &desc; + } +} + +void ArchiveManager::FileMap::shard_index_del(const FileDescription &desc) { + for (const auto &p : desc.first_blocks) { + ShardIndex &s = shards_[p.first]; + s.seqno_index_.erase(p.second.seqno); + s.lt_index_.erase(p.second.lt); + s.unix_time_index_.erase(p.second.ts); + s.packages_index_.erase(desc.id); + } +} + +void ArchiveManager::FileMap::set_shard_first_block(const FileDescription &desc, ShardIdFull shard, + FileDescription::Desc v) { + ShardIndex &s = shards_[shard]; + auto &d = const_cast(desc); + auto it = d.first_blocks.find(shard); + if (it != d.first_blocks.end()) { + s.seqno_index_.erase(it->second.seqno); + s.lt_index_.erase(it->second.lt); + s.unix_time_index_.erase(it->second.ts); + } + d.first_blocks[shard] = v; + s.seqno_index_[v.seqno] = &d; + s.lt_index_[v.lt] = &d; + s.unix_time_index_[v.ts] = &d; + s.packages_index_[d.id] = &d; +} + +const ArchiveManager::FileDescription *ArchiveManager::FileMap::get_file_desc_by_seqno(ShardIdFull shard, + BlockSeqno seqno) const { + auto it = shards_.find(shard); + if (it == shards_.end()) { + return nullptr; + } + const auto &map = it->second.seqno_index_; + auto it2 = map.upper_bound(seqno); + if (it2 == map.begin()) { + return nullptr; + } + --it2; + return it2->second->deleted ? nullptr : it2->second; +} + +const ArchiveManager::FileDescription *ArchiveManager::FileMap::get_file_desc_by_lt(ShardIdFull shard, + LogicalTime lt) const { + auto it = shards_.find(shard); + if (it == shards_.end()) { + return nullptr; + } + const auto &map = it->second.lt_index_; + auto it2 = map.upper_bound(lt); + if (it2 == map.begin()) { + return nullptr; + } + --it2; + return it2->second->deleted ? nullptr : it2->second; +} + +const ArchiveManager::FileDescription *ArchiveManager::FileMap::get_file_desc_by_unix_time(ShardIdFull shard, + UnixTime ts) const { + auto it = shards_.find(shard); + if (it == shards_.end()) { + return nullptr; + } + const auto &map = it->second.unix_time_index_; + auto it2 = map.upper_bound(ts); + if (it2 == map.begin()) { + return nullptr; + } + --it2; + return it2->second->deleted ? nullptr : it2->second; +} + +const ArchiveManager::FileDescription *ArchiveManager::FileMap::get_next_file_desc(ShardIdFull shard, + const FileDescription *desc) const { + auto it = shards_.find(shard); + if (it == shards_.end()) { + return nullptr; + } + const auto &map = it->second.packages_index_; + auto it2 = desc ? map.upper_bound(desc->id) : map.begin(); + if (it2 == map.end()) { + return nullptr; } - return false; + return it2->second->deleted ? nullptr : it2->second; } } // namespace validator diff --git a/validator/db/archive-manager.hpp b/validator/db/archive-manager.hpp index 79e6a2d78..e50087642 100644 --- a/validator/db/archive-manager.hpp +++ b/validator/db/archive-manager.hpp @@ -71,7 +71,6 @@ class ArchiveManager : public td::actor::Actor { void start_up() override; - void begin_transaction(); void commit_transaction(); void set_async_mode(bool mode, td::Promise promise); @@ -94,26 +93,83 @@ class ArchiveManager : public td::actor::Actor { auto file_actor_id() const { return file.get(); } - void clear_actor_id() { - file.reset(); - } - bool has_account_prefix(AccountIdPrefixFull account_id) const; PackageId id; - bool deleted; + mutable bool deleted; std::map first_blocks; - td::actor::ActorOwn file; + mutable td::actor::ActorOwn file; }; - std::map files_; - std::map key_files_; - std::map temp_files_; + class FileMap { + public: + std::map::const_iterator begin() const { + return files_.cbegin(); + } + std::map::const_iterator end() const { + return files_.cend(); + } + std::map::const_reverse_iterator rbegin() const { + return files_.crbegin(); + } + std::map::const_reverse_iterator rend() const { + return files_.crend(); + } + std::map::const_iterator find(PackageId x) const { + return files_.find(x); + } + size_t count(const PackageId &x) const { + return files_.count(x); + } + size_t size() const { + return files_.size(); + } + std::map::const_iterator lower_bound(const PackageId &x) const { + return files_.lower_bound(x); + } + std::map::const_iterator upper_bound(const PackageId &x) const { + return files_.upper_bound(x); + } + void clear() { + files_.clear(); + shards_.clear(); + } + const FileDescription &emplace(const PackageId &id, FileDescription desc) { + auto it = files_.emplace(id, std::move(desc)); + if (it.second) { + shard_index_add(it.first->second); + } + return it.first->second; + } + void erase(std::map::const_iterator it) { + shard_index_del(it->second); + files_.erase(it); + } + void set_shard_first_block(const FileDescription &desc, ShardIdFull shard, FileDescription::Desc v); + const FileDescription *get_file_desc_by_seqno(ShardIdFull shard, BlockSeqno seqno) const; + const FileDescription *get_file_desc_by_lt(ShardIdFull shard, LogicalTime lt) const; + const FileDescription *get_file_desc_by_unix_time(ShardIdFull shard, UnixTime ts) const; + const FileDescription *get_next_file_desc(ShardIdFull shard, const FileDescription *desc) const; + + private: + std::map files_; + struct ShardIndex { + std::map seqno_index_; + std::map lt_index_; + std::map unix_time_index_; + std::map packages_index_; + }; + std::map shards_; + + void shard_index_add(const FileDescription &desc); + void shard_index_del(const FileDescription &desc); + }; + FileMap files_, key_files_, temp_files_; BlockSeqno finalized_up_to_{0}; bool async_mode_ = false; bool huge_transaction_started_ = false; td::uint32 huge_transaction_size_ = 0; - auto &get_file_map(const PackageId &p) { + FileMap &get_file_map(const PackageId &p) { return p.key ? key_files_ : p.temp ? temp_files_ : files_; } @@ -126,18 +182,19 @@ class ArchiveManager : public td::actor::Actor { void get_handle_finish(BlockHandle handle, td::Promise promise); void get_file_short_cont(FileReference ref_id, PackageId idx, td::Promise promise); - FileDescription *get_file_desc(ShardIdFull shard, PackageId id, BlockSeqno seqno, UnixTime ts, LogicalTime lt, - bool force); - FileDescription *add_file_desc(ShardIdFull shard, PackageId id, BlockSeqno seqno, UnixTime ts, LogicalTime lt); - void update_desc(FileDescription &desc, ShardIdFull shard, BlockSeqno seqno, UnixTime ts, LogicalTime lt); - FileDescription *get_file_desc_by_seqno(ShardIdFull shard, BlockSeqno seqno, bool key_block); - FileDescription *get_file_desc_by_lt(ShardIdFull shard, LogicalTime lt, bool key_block); - FileDescription *get_file_desc_by_unix_time(ShardIdFull shard, UnixTime ts, bool key_block); - FileDescription *get_file_desc_by_seqno(AccountIdPrefixFull shard, BlockSeqno seqno, bool key_block); - FileDescription *get_file_desc_by_lt(AccountIdPrefixFull shard, LogicalTime lt, bool key_block); - FileDescription *get_file_desc_by_unix_time(AccountIdPrefixFull shard, UnixTime ts, bool key_block); - FileDescription *get_next_file_desc(FileDescription *f); - FileDescription *get_temp_file_desc_by_idx(PackageId idx); + const FileDescription *get_file_desc(ShardIdFull shard, PackageId id, BlockSeqno seqno, UnixTime ts, LogicalTime lt, + bool force); + const FileDescription *add_file_desc(ShardIdFull shard, PackageId id, BlockSeqno seqno, UnixTime ts, LogicalTime lt); + void update_desc(FileMap &f, const FileDescription &desc, ShardIdFull shard, BlockSeqno seqno, UnixTime ts, + LogicalTime lt); + const FileDescription *get_file_desc_by_seqno(ShardIdFull shard, BlockSeqno seqno, bool key_block); + const FileDescription *get_file_desc_by_lt(ShardIdFull shard, LogicalTime lt, bool key_block); + const FileDescription *get_file_desc_by_unix_time(ShardIdFull shard, UnixTime ts, bool key_block); + const FileDescription *get_file_desc_by_seqno(AccountIdPrefixFull shard, BlockSeqno seqno, bool key_block); + const FileDescription *get_file_desc_by_lt(AccountIdPrefixFull shard, LogicalTime lt, bool key_block); + const FileDescription *get_file_desc_by_unix_time(AccountIdPrefixFull shard, UnixTime ts, bool key_block); + const FileDescription *get_next_file_desc(const FileDescription *f, AccountIdPrefixFull shard, bool key_block); + const FileDescription *get_temp_file_desc_by_idx(PackageId idx); PackageId get_max_temp_file_desc_idx(); PackageId get_prev_temp_file_desc_idx(PackageId id);