From cbb853e9231351a7a74bcd6c8ad297265ee1a824 Mon Sep 17 00:00:00 2001 From: Alan Sapede Date: Mon, 15 May 2023 18:25:05 +0200 Subject: [PATCH] Improve github actions (#2300) * More memory for node * Refactor github action with template (WIP) * Fixes coverage for master comparison * fixes path to template * better name for template * adds checkout before other actions * maybe * Fixes composite * Adds required shell for composite * Adds name to template build * Cleanup variables * changing require method in typescript api * better json parsing * adds log * pushing to use node 20 * Adds logs * Updates node to 20.x * better cancel * Split typescript test into separate workflow * Moves all to node 18 * Fixes package lock check * Adds mold ? * adds better mold * fixes build workflow syntax * adds log * Better setup for mold * fix mold path * fix cov too * fixes mold again * better rustflags * Update build.yml * Update action.yml * Update action.yml * Better env vars * Update action.yml * better diff check * Update action.yml * correct variable check * Update action.yml * testing less parallelism in tests * reduce more * fixes removed timeout * let default mocha take care of timeout * Reduce parallel speed * Increase timeout even more * Force coverage to execute * Adds wait for Eth * Testing more parallel tests * adds file du * Remove useless profraws * Forces to pass for now * typo * Increased timeout for default cases * 30s timeout for ts * Fixes provider test issues * Restore port * fix added logs --- .../workflow-templates/cargo-build/action.yml | 78 +++++++++ .../typescript-tests/action.yml | 107 ++++++++++++ .github/workflows/build.yml | 163 +++--------------- .github/workflows/cancel.yml | 2 +- .github/workflows/client-release-issue.yml | 4 +- .github/workflows/coverage.yml | 96 +++-------- .github/workflows/publish-binary.yml | 4 +- .github/workflows/publish-runtime.yml | 4 +- .github/workflows/publish-typescript-api.yml | 8 +- .github/workflows/runtime-release-issue.yml | 4 +- .github/workflows/upgrade-typescript-api.yml | 4 +- .github/workflows/version-bump.yml | 4 +- tests/package.json | 2 +- tests/util/setup-dev-tests.ts | 48 ++++-- typescript-api/scripts/postbuild.js | 13 +- 15 files changed, 304 insertions(+), 237 deletions(-) create mode 100644 .github/workflow-templates/cargo-build/action.yml create mode 100644 .github/workflow-templates/typescript-tests/action.yml diff --git a/.github/workflow-templates/cargo-build/action.yml b/.github/workflow-templates/cargo-build/action.yml new file mode 100644 index 0000000000..29e463572b --- /dev/null +++ b/.github/workflow-templates/cargo-build/action.yml @@ -0,0 +1,78 @@ +name: Cargo build +description: | + Builds moonbeam with given features. + Stores the result in "build/moonbeam" and the runtimes in "runtimes/" + +inputs: + features: + description: features to include in the build (comma separated) + required: false + +env: + RUSTFLAGS: "-C opt-level=3 -D warnings -C linker=clang -C link-arg=-fuse-ld=./mold/bin/mold" + RUSTC_WRAPPER: "sccache" + CARGO_INCREMENTAL: "0" + SCCACHE_CACHE_SIZE: "100GB" + +runs: + using: "composite" + steps: + - name: Run sccache-cache + uses: mozilla-actions/sccache-action@v0.0.3 + - name: Setup Variables + shell: bash + run: | + echo "CARGO_INCREMENTAL=0" >> $GITHUB_ENV + echo "RUSTC_WRAPPER=sccache" >> $GITHUB_ENV + echo "SCCACHE_CACHE_SIZE=100GB" >> $GITHUB_ENV + # Set RUSTFLAGS if not already set + if [ -z "$RUSTFLAGS" ]; then + echo "RUSTFLAGS=-C opt-level=3 -D warnings -C linker=clang -C link-arg=-fuse-ld=$(pwd)/mold/bin/mold" >> $GITHUB_ENV + fi + - name: Setup Mold Linker + shell: bash + run: | + mkdir -p mold + curl -L --retry 10 --silent --show-error https://github.com/rui314/mold/releases/download/v1.1.1/mold-1.1.1-$(uname -m)-linux.tar.gz | tar -C $(realpath mold) --strip-components=1 -xzf - + # With rustup's nice new toml format, we just need to run rustup show to install the toolchain + # https://github.com/actions-rs/toolchain/issues/126#issuecomment-782989659 + - name: Setup Rust toolchain + shell: bash + run: | + if ! which "rustup" > /dev/null; then + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y + fi + rustup show + - name: Build Node + shell: bash + run: | + env + params=" --locked --release -p moonbeam" + if [ -n "${{ github.event.inputs.features }}" ]; then + params="$params --features ${{ github.event.inputs.features }}" + fi + echo "cargo build $params" + cargo build $params + - name: Display binary comments + shell: bash + run: readelf -p .comment ./target/release/moonbeam + - name: Display sccache stats + shell: bash + run: ${SCCACHE_PATH} --show-stats + - name: Verify binary version + shell: bash + run: | + GIT_COMMIT=`git log -1 --format="%H" | cut -c1-7` + MB_VERSION=`./target/release/moonbeam --version` + echo "Checking $MB_VERSION contains $GIT_COMMIT" + echo "$MB_VERSION" | grep $GIT_COMMIT + - name: Save runtimes wasm + shell: bash + run: | + mkdir -p runtimes + cp target/release/wbuild/moon*/moon*_runtime.compact.compressed.wasm runtimes/; + - name: Save moonbeam binary + shell: bash + run: | + mkdir -p build + cp target/release/moonbeam build/moonbeam; diff --git a/.github/workflow-templates/typescript-tests/action.yml b/.github/workflow-templates/typescript-tests/action.yml new file mode 100644 index 0000000000..e546788ce6 --- /dev/null +++ b/.github/workflow-templates/typescript-tests/action.yml @@ -0,0 +1,107 @@ +name: Typescript tests +description: | + Setup and run the typescript tests against a local moonbeam node. + +inputs: + moonbeam-binary: + description: path to the moonbeam binary + required: true + timeout: + description: default timeout for the tests + default: 5000 + force-pass: + description: If the tests should pass even if there are failures + default: false + +env: + NODE_OPTIONS: "--max-old-space-size=12288" + +runs: + using: "composite" + steps: + - name: Use Node.js 18.x + uses: actions/setup-node@v3 + with: + node-version: 18.x + - name: Set binary path and number of CPUs + shell: bash + run: | + #### Ensure the moonbeam binary is executable + chmod uog+x ${{ inputs.moonbeam-binary }} + + #### Set the binary path (absolute path to avoid subfolder working directories issues) + echo "BINARY_PATH=$(realpath ${{ inputs.moonbeam-binary }})" >> $GITHUB_ENV + + #### Set the number of CPUs to use for parallel tests. + echo "CPUS=$(lscpu | egrep '^CPU\(s\)' | grep -o '[0-9]*')" >> $GITHUB_ENV + + echo "BINARY_PATH: $BINARY_PATH" + echo "CPUS: $CPUS" + + - name: Setup Moonbeam PolkadotJS types + shell: bash + run: | + #### Preparing the legacy types + cd moonbeam-types-bundle + npm ci + npm run build + + #### Preparing the typescript api + cd ../typescript-api + npm ci + + cd ../tests + npm ci + + #### Prepares and copies the typescript generated API to include in the tests + npm run setup-typescript-api + + - name: Compile Typescript tests + shell: bash + run: | + #### Compile typescript tests into javascript (more stable for Mocha) + #### This also better display typescript issues + cd ./tests + npm run build + + - name: Execute tests + shell: bash + run: | + cd ./tests + node node_modules/.bin/mocha \ + --timeout ${{ inputs.timeout }} \ + --parallel -j $((CPUS / 4)) \ + --exit \ + 'build/tests/**/test-*.js' || ${{ inputs.force-pass }} + + # We determine whether there are unmodified package-lock.json files by: + # 1. Asking git for a list of all modified files + # 2. Using grep to reduce the list to only package-lock.json files + # 3. Counting the number of lines of output + - name: Check package-lock.json + shell: bash + run: | + cd ./tests + # Log npm version to make sure it maches with local version + npm -v + # Make sure git is working, and if not abort early. When git is not working it looks like: + # $ git diff-index --name-only HEAD + # fatal: not a git repository (or any of the parent directories): .git + DIFF_INDEX="$(git diff-index --name-only HEAD)" + echo $DIFF_INDEX + if [[ -n "$DIFF_INDEX" ]]; then + if [[ ${DIFF_INDEX:0:5} == "fatal" ]]; then + echo "There was an error with the git checkout. Can't check package-lock.json file." + false + fi + FILECOUNT=$(echo $DIFF_INDEX | grep package-lock.json | wc -l) + echo $FILECOUNT + if [[ $FILECOUNT -eq 0 ]]; then + echo "All package-lock.json files are valid" + else + git diff --cached + echo "The following package-lock.json files have uncommitted changes" + echo $DIFF_INDEX | grep package-lock.json + false + fi + fi diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 273d60abd8..be4ab55b54 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -16,13 +16,12 @@ on: pull_request: description: set to pull_request number to execute on external pr required: false + env: - SCCACHE_CACHE_SIZE: "100GB" - CARGO_INCREMENTAL: "0" + NODE_OPTIONS: "--max-old-space-size=12288" jobs: ####### Check files and formatting ####### - set-tags: runs-on: ubuntu-latest outputs: @@ -146,10 +145,10 @@ jobs: uses: actions/checkout@v3 with: ref: ${{ needs.set-tags.outputs.git_ref }} - - name: Use Node.js 16.x + - name: Use Node.js 18.x uses: actions/setup-node@v3 with: - node-version: 16.x + node-version: 18.x - name: Check with Prettier run: npx prettier --check --ignore-path .prettierignore '**/*.(yml|js|ts|json)' @@ -280,52 +279,18 @@ jobs: runs-on: labels: bare-metal needs: ["set-tags"] - env: - RUSTFLAGS: "-C opt-level=3 -D warnings" - RUSTC_WRAPPER: "sccache" - TMP_TARGET: "/tmp/target" - CARGO_TARGET_DIR: "target" steps: - name: Checkout uses: actions/checkout@v3 with: ref: ${{ needs.set-tags.outputs.git_ref }} - - name: Run sccache-cache - uses: mozilla-actions/sccache-action@v0.0.3 - # With rustup's nice new toml format, we just need to run rustup show to install the toolchain - # https://github.com/actions-rs/toolchain/issues/126#issuecomment-782989659 - - name: Setup Rust toolchain - run: | - if ! which "rustup" > /dev/null; then - curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y - fi - rustup show - - name: Build Node - run: | - env - cargo build --locked --release -p moonbeam - - name: Run sccache stat for check pre test - shell: bash - run: ${SCCACHE_PATH} --show-stats - - name: Verify node version - run: | - GIT_COMMIT=`git log -1 --format="%H" | cut -c1-7` - MB_VERSION=`./$CARGO_TARGET_DIR/release/moonbeam --version` - echo "Checking $MB_VERSION contains $GIT_COMMIT" - echo "$MB_VERSION" | grep $GIT_COMMIT - - name: Save runtime wasm - run: | - mkdir -p runtimes - cp $CARGO_TARGET_DIR/release/wbuild/moon*/moon*_runtime.compact.compressed.wasm runtimes/; + - name: Cargo build + uses: ./.github/workflow-templates/cargo-build - name: Upload runtimes uses: actions/upload-artifact@v3.1.2 with: name: runtimes path: runtimes - - name: Save moonbeam binary - run: | - mkdir -p build - cp $CARGO_TARGET_DIR/release/moonbeam build/moonbeam; - name: Upload binary uses: actions/upload-artifact@v3.1.2 with: @@ -336,43 +301,15 @@ jobs: runs-on: labels: bare-metal needs: ["set-tags"] - env: - RUSTFLAGS: "-C opt-level=3 -D warnings" - RUSTC_WRAPPER: "sccache" - TMP_TARGET: "/tmp/target-features" - CARGO_TARGET_DIR: "target-features" steps: - name: Checkout uses: actions/checkout@v3 with: ref: ${{ needs.set-tags.outputs.git_ref }} - - name: Run sccache-cache - uses: mozilla-actions/sccache-action@v0.0.3 - # With rustup's nice new toml format, we just need to run rustup show to install the toolchain - # https://github.com/actions-rs/toolchain/issues/126#issuecomment-782989659 - - name: Setup Rust toolchain - run: | - if ! which "rustup" > /dev/null; then - curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y - fi - rustup show - - name: Build Node - run: | - env - cargo build --locked --release --features=try-runtime,runtime-benchmarks - - name: Run sccache stat for check pre test - shell: bash - run: ${SCCACHE_PATH} --show-stats - - name: Verify node version - run: | - GIT_COMMIT=`git log -1 --format="%H" | cut -c1-7` - MB_VERSION=`./$CARGO_TARGET_DIR/release/moonbeam --version` - echo "Checking $MB_VERSION contains $GIT_COMMIT" - echo "$MB_VERSION" | grep $GIT_COMMIT - - name: Save moonbeam binary - run: | - mkdir -p build - cp $CARGO_TARGET_DIR/release/moonbeam build/moonbeam-features; + - name: Cargo build + uses: ./.github/workflow-templates/cargo-build + with: + features: "try-runtime,runtime-benchmarks" - name: Upload binary uses: actions/upload-artifact@v3.1.2 with: @@ -384,10 +321,9 @@ jobs: labels: bare-metal needs: ["set-tags"] env: - RUSTFLAGS: "-C opt-level=3 -D warnings" RUSTC_WRAPPER: "sccache" - TMP_TARGET: "/tmp/target-test" - CARGO_TARGET_DIR: "target-test" + CARGO_INCREMENTAL: "0" + SCCACHE_CACHE_SIZE: "100GB" steps: - name: Checkout uses: actions/checkout@v3 @@ -395,6 +331,15 @@ jobs: ref: ${{ needs.set-tags.outputs.git_ref }} - name: Run sccache-cache uses: mozilla-actions/sccache-action@v0.0.3 + - name: Setup Variables + shell: bash + run: | + echo "RUSTFLAGS=-C opt-level=3 -D warnings -C linker=clang -C link-arg=-fuse-ld=$(pwd)/mold/bin/mold" >> $GITHUB_ENV + - name: Setup Mold Linker + shell: bash + run: | + mkdir -p mold + curl -L --retry 10 --silent --show-error https://github.com/rui314/mold/releases/download/v1.1.1/mold-1.1.1-$(uname -m)-linux.tar.gz | tar -C $(realpath mold) --strip-components=1 -xzf - # With rustup's nice new toml format, we just need to run rustup show to install the toolchain # https://github.com/actions-rs/toolchain/issues/126#issuecomment-782989659 - name: Setup Rust toolchain @@ -408,7 +353,6 @@ jobs: run: | cargo test --profile testnet --all --features=evm-tracing - name: Run sccache stat for check pre test - shell: bash run: ${SCCACHE_PATH} --show-stats typescript-tests: @@ -426,64 +370,11 @@ jobs: with: name: moonbeam path: build - - name: Use Node.js 20.x - uses: actions/setup-node@v3 + - name: Typescript Tests (Dev Service) + uses: ./.github/workflow-templates/typescript-tests with: - node-version: 20.x - - name: Typescript integration tests (against dev service) - run: | - chmod uog+x build/moonbeam - - #### Preparing the repository - cd moonbeam-types-bundle - npm ci - npm run build - - #### Preparing the typescript api - cd ../typescript-api - npm ci - - cd ../tests - npm ci - #### Prepares and copies the typescript generated API to include in the tests - npm run setup-typescript-api - - #### Compile typescript tests into javascript (more stable for Mocha) - #### This also better display typescript issues - npm run build - - CPUS=$(lscpu | egrep '^CPU\(s\)' | grep -o '[0-9]*') - - node node_modules/.bin/mocha \ - --parallel -j $((CPUS / 2)) \ - --exit \ - 'build/tests/**/test-*.js' - - # We determine whether there are unmodified package-lock.json files by: - # 1. Asking git for a list of all modified files - # 2. Using grep to reduce the list to only package-lock.json files - # 3. Counting the number of lines of output - - name: Check package-lock.json - run: | - # Log npm version to make sure it maches with local version - npm -v - # Make sure git is working, and if not abort early. When git is not working it looks like: - # $ git diff-index --name-only HEAD - # fatal: not a git repository (or any of the parent directories): .git - DIFF_INDEX=$(git diff-index --name-only HEAD) - if [[ ${DIFF_INDEX:0:5} == "fatal" ]]; then - echo "There was an error with the git checkout. Can't check package-lock.json file." - false - fi - FILECOUNT=$(echo $DIFF_INDEX | grep package-lock.json | wc -l) - if [[ $FILECOUNT -eq 0 ]]; then - echo "All package-lock.json files are valid" - else - git diff --cached - echo "The following package-lock.json files have uncommitted changes" - echo $DIFF_INDEX | grep package-lock.json - false - fi + moonbeam-binary: build/moonbeam + timeout: 30000 typescript-tracing-tests: if: github.ref == 'refs/heads/master' || contains(github.event.pull_request.labels.*.name, 'A10-evmtracing') @@ -499,10 +390,10 @@ jobs: with: name: moonbeam path: build - - name: Use Node.js 16.x + - name: Use Node.js 18.x uses: actions/setup-node@v3 with: - node-version: 16.x + node-version: 18.x - name: Get tracing runtimes run: | ./scripts/build-last-tracing-runtime.sh ${{ needs.set-tags.outputs.git_branch }} diff --git a/.github/workflows/cancel.yml b/.github/workflows/cancel.yml index 179f8c90e8..ba9202ddd1 100644 --- a/.github/workflows/cancel.yml +++ b/.github/workflows/cancel.yml @@ -9,6 +9,6 @@ jobs: steps: - uses: styfle/cancel-workflow-action@0.9.1 with: - workflow_id: ".github/workflows/build.yml" + workflow_id: ".github/workflows/build.yml,.github/workflows/coverage.yml" all_but_latest: true access_token: ${{ github.token }} diff --git a/.github/workflows/client-release-issue.yml b/.github/workflows/client-release-issue.yml index 46d4aefec6..30781e96f4 100644 --- a/.github/workflows/client-release-issue.yml +++ b/.github/workflows/client-release-issue.yml @@ -26,10 +26,10 @@ jobs: steps: - name: Checkout uses: actions/checkout@v3 - - name: Use Node.js 16.x + - name: Use Node.js 18.x uses: actions/setup-node@v3 with: - node-version: 16.x + node-version: 18.x - name: Download Original Tools uses: actions/download-artifact@v3 with: diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index ebe765e9bf..9fab0bc3c3 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -10,9 +10,9 @@ on: push: branches: - master + env: - SCCACHE_CACHE_SIZE: "100GB" - CARGO_INCREMENTAL: "0" + NODE_OPTIONS: "--max-old-space-size=12288" jobs: ####### Check files and formatting ####### @@ -89,24 +89,18 @@ jobs: labels: bare-metal needs: ["set-tags"] env: - RUSTFLAGS: "-C opt-level=3 -D warnings -C instrument-coverage" RUSTC_WRAPPER: "sccache" - CARGO_TARGET_DIR: "target" + CARGO_INCREMENTAL: "0" + SCCACHE_CACHE_SIZE: "100GB" steps: - name: Checkout uses: actions/checkout@v3 with: ref: ${{ needs.set-tags.outputs.git_ref }} - - name: Run sccache-cache - uses: mozilla-actions/sccache-action@v0.0.3 - # With rustup's nice new toml format, we just need to run rustup show to install the toolchain - # https://github.com/actions-rs/toolchain/issues/126#issuecomment-782989659 - - name: Setup Rust toolchain + - name: Setup Variables + shell: bash run: | - if ! which "rustup" > /dev/null; then - curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y - fi - rustup show + echo "RUSTFLAGS=-C opt-level=3 -D warnings -C instrument-coverage -C linker=clang -C link-arg=-fuse-ld=$(pwd)/mold/bin/mold" >> $GITHUB_ENV - name: Setup grcov run: | wget https://alan-stuff.s3.amazonaws.com/grcov -O grcov @@ -117,32 +111,18 @@ jobs: wget https://alan-stuff.s3.amazonaws.com/llvm-cov -O llvm-cov chmod +x llvm-profdata chmod +x llvm-cov - - name: Build Node - run: | - env - cargo build --locked --release -p moonbeam - - name: Run sccache stat for check pre test - shell: bash - run: ${SCCACHE_PATH} --show-stats - - name: Verify node version - run: | - GIT_COMMIT=`git log -1 --format="%H" | cut -c1-7` - MB_VERSION=`./$CARGO_TARGET_DIR/release/moonbeam --version` - echo "Checking $MB_VERSION contains $GIT_COMMIT" - echo "$MB_VERSION" | grep $GIT_COMMIT - - name: Save runtime wasm + - name: Cargo build + uses: ./.github/workflow-templates/cargo-build + with: + features: evm-tracing + - name: Clean-up possible coverage generated during builds run: | - mkdir -p runtimes - cp $CARGO_TARGET_DIR/release/wbuild/moon*/moon*_runtime.compact.compressed.wasm runtimes/; + rm default_*.profraw - name: Upload runtimes uses: actions/upload-artifact@v3.1.2 with: name: runtimes path: runtimes - - name: Save moonbeam binary - run: | - mkdir -p build - cp $CARGO_TARGET_DIR/release/moonbeam build/moonbeam; - name: Upload binary uses: actions/upload-artifact@v3.1.2 with: @@ -151,56 +131,28 @@ jobs: - name: Unit tests run: | cargo test --release --all --features=evm-tracing - - name: Use Node.js 20.x - uses: actions/setup-node@v3 + - name: Typescript Tests (Dev Service) + uses: ./.github/workflow-templates/typescript-tests with: - node-version: 20.x - - name: Typescript integration tests (against dev service) - run: | - chmod uog+x build/moonbeam - - #### Preparing the repository - cd moonbeam-types-bundle - npm ci - npm run build - - #### Preparing the typescript api - cd ../typescript-api - npm ci - - cd ../tests - npm ci - #### Prepares and copies the typescript generated API to include in the tests - npm run setup-typescript-api - - #### Compile typescript tests into javascript (more stable for Mocha) - #### This also better display typescript issues - npm run build - - CPUS=$(lscpu | egrep '^CPU\(s\)' | grep -o '[0-9]*') - - node node_modules/.bin/mocha \ - --timeout 180000 \ - --parallel -j $((CPUS / 2)) \ - --exit \ - 'build/tests/**/test-*.js' || true - # We force it to pass to get the coverage report + timeout: 10000 + moonbeam-binary: build/moonbeam + force-pass: true - name: Retrieve coverage id: coverage run: | mkdir -p /tmp/proffiles - echo "Deletes invalid raw files (<74Mb) not suze why we have them" find . -type f -name \*.profraw -exec ls -l {} \; echo "Copying profraw files to /tmp/proffiles" find . -name \*.profraw -exec mv {} /tmp/proffiles/ \; mv /tmp/proffiles proffiles + du -sh proffiles echo "Executing grcov" - ./grcov proffiles/ -s ./ --binary-path ./${CARGO_TARGET_DIR}/release/ \ - -t html --branch --ignore-not-existing --ignore ${CARGO_TARGET_DIR}/release/build/* \ + ./grcov proffiles/ -s ./ --binary-path ./target/release/ \ + -t html --branch --ignore-not-existing --ignore target/release/build/* \ -o coverage/ --llvm-path ./ --llvm 2>&1 \ | tee grcov.log @@ -213,8 +165,8 @@ jobs: cd proffiles/ rm $INVALID cd .. - ./grcov proffiles/ -s ./ --binary-path ./${CARGO_TARGET_DIR}/release/ \ - -t html --branch --ignore-not-existing --ignore ${CARGO_TARGET_DIR}/release/build/* \ + ./grcov proffiles/ -s ./ --binary-path ./target/release/ \ + -t html --branch --ignore-not-existing --ignore target/release/build/* \ -o coverage/ --llvm-path ./ --llvm fi @@ -222,7 +174,7 @@ jobs: echo "total_percent=$(grep -o '[0-9\.]*%' coverage/html/coverage.json)" >> $GITHUB_OUTPUT wget https://${{ vars.S3_COVERAGE_BUCKET }}.s3.amazonaws.com/branches/master/html/coverage.json \ -O coverage-master.json || true - echo "master_percent=$(grep -o '[0-9\.]*%' coverage/html/coverage-master.json || echo 'N/A')" >> $GITHUB_OUTPUT + echo "master_percent=$(grep -o '[0-9\.]*%' coverage-master.json || echo 'N/A')" >> $GITHUB_OUTPUT rm -rf proffiles/ - name: Upload coverate to gha uses: actions/upload-artifact@v3.1.2 diff --git a/.github/workflows/publish-binary.yml b/.github/workflows/publish-binary.yml index a266e6905c..a7696246f0 100644 --- a/.github/workflows/publish-binary.yml +++ b/.github/workflows/publish-binary.yml @@ -74,10 +74,10 @@ jobs: with: name: moonbeam path: build - - name: Use Node.js 16.x + - name: Use Node.js 18.x uses: actions/setup-node@v3 with: - node-version: 16.x + node-version: 18.x - name: Download Original Tools uses: actions/download-artifact@v3 with: diff --git a/.github/workflows/publish-runtime.yml b/.github/workflows/publish-runtime.yml index a2817975dc..d75da9e236 100644 --- a/.github/workflows/publish-runtime.yml +++ b/.github/workflows/publish-runtime.yml @@ -122,10 +122,10 @@ jobs: with: name: moonbeam-runtime path: build - - name: Use Node.js 16.x + - name: Use Node.js 18.x uses: actions/setup-node@v3 with: - node-version: 16.x + node-version: 18.x - name: Download Original Tools uses: actions/download-artifact@v3 with: diff --git a/.github/workflows/publish-typescript-api.yml b/.github/workflows/publish-typescript-api.yml index 5b95aa60a9..6f32f2c045 100644 --- a/.github/workflows/publish-typescript-api.yml +++ b/.github/workflows/publish-typescript-api.yml @@ -14,10 +14,10 @@ jobs: uses: actions/checkout@v3 with: ref: ${{ github.event.inputs.sha }} - - name: Use Node.js 16.x + - name: Use Node.js 18.x uses: actions/setup-node@v3 with: - node-version: 16.x + node-version: 18.x - name: Build typescript API run: | cd typescript-api @@ -37,10 +37,10 @@ jobs: uses: actions/checkout@v3 with: ref: ${{ github.event.inputs.sha }} - - name: Use Node.js 16.x + - name: Use Node.js 18.x uses: actions/setup-node@v3 with: - node-version: 16.x + node-version: 18.x - name: Upgrade polkadotjs for tests run: | cd tests diff --git a/.github/workflows/runtime-release-issue.yml b/.github/workflows/runtime-release-issue.yml index 016935c379..04b51219db 100644 --- a/.github/workflows/runtime-release-issue.yml +++ b/.github/workflows/runtime-release-issue.yml @@ -29,10 +29,10 @@ jobs: steps: - name: Checkout uses: actions/checkout@v3 - - name: Use Node.js 16.x + - name: Use Node.js 18.x uses: actions/setup-node@v3 with: - node-version: 16.x + node-version: 18.x - name: Download Original Tools uses: actions/download-artifact@v3 with: diff --git a/.github/workflows/upgrade-typescript-api.yml b/.github/workflows/upgrade-typescript-api.yml index 8dccddccb1..f6686bb64e 100644 --- a/.github/workflows/upgrade-typescript-api.yml +++ b/.github/workflows/upgrade-typescript-api.yml @@ -24,10 +24,10 @@ jobs: mkdir -p build docker cp dummy:/moonbeam/moonbeam build/moonbeam docker rm -f dummy - - name: Use Node.js 16.x + - name: Use Node.js 18.x uses: actions/setup-node@v3 with: - node-version: 16.x + node-version: 18.x - name: Upgrade polkadotjs for moonbeam-types-bundle run: | cd moonbeam-types-bundle diff --git a/.github/workflows/version-bump.yml b/.github/workflows/version-bump.yml index 9668a797da..96621b9fae 100644 --- a/.github/workflows/version-bump.yml +++ b/.github/workflows/version-bump.yml @@ -15,10 +15,10 @@ jobs: steps: - name: Checkout uses: actions/checkout@v3 - - name: Use Node.js 16.x + - name: Use Node.js 18.x uses: actions/setup-node@v3 with: - node-version: 16.x + node-version: 18.x - name: Generate version bump issue env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/tests/package.json b/tests/package.json index f14d7fa8ec..4b38e81d99 100644 --- a/tests/package.json +++ b/tests/package.json @@ -31,7 +31,7 @@ }, "scripts": { "test-with-logs": "mocha --printlogs -r ts-node/register 'tests/**/test-*.ts'", - "setup-typescript-api": "rm -rf node_modules/@moonbeam-network/api-augment && cd ../typescript-api && npm run build && cp -r build ../tests/node_modules/@moonbeam-network/api-augment", + "setup-typescript-api": "rm -rf node_modules/@moonbeam-network/api-augment && cd ../typescript-api && npm run build && cp -r build ../tests/node_modules/@moonbeam-network/api-augment && echo 'Typescript API Local installed!'", "pre-build-contracts": "ts-node tools/pre-build-contracts.ts && npx prettier --write ./contracts/compiled/", "test": "mocha --parallel -r ts-node/register 'tests/**/test-*.ts' -- -j 4", "test-seq": "mocha -r ts-node/register 'tests/**/test-*.ts'", diff --git a/tests/util/setup-dev-tests.ts b/tests/util/setup-dev-tests.ts index 6e90608b5e..422810a621 100644 --- a/tests/util/setup-dev-tests.ts +++ b/tests/util/setup-dev-tests.ts @@ -84,9 +84,6 @@ export function describeDevMoonbeam( forkedMode?: boolean ) { describe(title, function () { - // Set timeout to 5000 for all tests. - this.timeout(5000); - // The context is initialized empty to allow passing a reference // and to be filled once the node information is retrieved let context: InternalDevTestContext = { ethTransactionType } as InternalDevTestContext; @@ -140,9 +137,13 @@ export function describeDevMoonbeam( return apiPromise; }; - context.polkadotApi = await context.createPolkadotApi(); - context.web3 = await context.createWeb3(); - context.ethers = await context.createEthers(); + let subProvider: EnhancedWeb3; + [context.polkadotApi, context.web3, context.ethers, subProvider] = await Promise.all([ + context.createPolkadotApi(), + context.createWeb3(), + context.createEthers(), + context.createWeb3("ws"), + ]); context.createBlock = async < ApiType extends ApiTypes, @@ -197,6 +198,35 @@ export function describeDevMoonbeam( } const { parentHash, finalize } = options; + + // We are now listening to the eth block too. The main reason is because the Ethereum + // ingestion in Frontier is asynchronous, and can sometime be slightly delayed. This + // generates some race condition if we don't wait for it. + + let expectedBlockNumber: number = (await subProvider.eth.getBlockNumber()) + 1; + const ethCheckPromise = new Promise((resolve) => { + const ethBlockSub = subProvider.eth + .subscribe("newBlockHeaders", function (error, result) { + if (!error) { + return; + } + console.error(error); + }) + .on("data", function (blockHeader) { + // unsubscribes the subscription once we get the right block + if (blockHeader.number != expectedBlockNumber) { + debug( + `Received unexpected block: ${blockHeader.number} ` + + `(expected: ${expectedBlockNumber})` + ); + return; + } + ethBlockSub.unsubscribe(); + resolve(); + }) + .on("error", console.error); + }); + const blockResult = await createAndFinalizeBlock(context.polkadotApi, parentHash, finalize); // No need to extract events if no transactions @@ -244,11 +274,9 @@ export function describeDevMoonbeam( hash: result.hash, }; }); + // Ensure Ethereum block is also ready + await ethCheckPromise; - // Adds extra time to avoid empty transaction when querying it - if (results.find((r) => r.type == "eth")) { - await new Promise((resolve) => setTimeout(resolve, 2)); - } return { block: blockResult, result: Array.isArray(transactions) ? result : (result[0] as any), diff --git a/typescript-api/scripts/postbuild.js b/typescript-api/scripts/postbuild.js index 6692869b0a..d151c80c80 100644 --- a/typescript-api/scripts/postbuild.js +++ b/typescript-api/scripts/postbuild.js @@ -1,5 +1,13 @@ import { writeFileSync, copyFileSync } from "fs"; -import pck from "../package.json" assert { type: "json" }; +import { readFile } from 'fs/promises'; + +console.log("Loading package.json"); + +const pck = JSON.parse( + await readFile( + new URL('../package.json', import.meta.url) + ) +); const buildPath = `${process.env.PWD}/build`; @@ -8,10 +16,13 @@ pck.private = false; pck.type = "module"; pck.files = ["**/*", "!**/tsconfig.tsbuildinfo", "!**/*.tgz"]; +console.log(`Writing ${buildPath}/package.json`); writeFileSync(`${buildPath}/package.json`, JSON.stringify(pck, null, 2)); copyFileSync("README.md", `${buildPath}/README.md`); +console.log(`Copy ${buildPath}/README.md`); // Copy empty files for CommonJS modules copyFileSync("./src/index.cjs", `${buildPath}/index.cjs`); copyFileSync("./src/index.cjs", `${buildPath}/moonriver/index.cjs`); copyFileSync("./src/index.cjs", `${buildPath}/moonbase/index.cjs`); +console.log(`Done postbuild`);