Improve github actions (#2300)

* More memory for node

* Refactor github action with template (WIP)

* Fixes coverage for master comparison

* fixes path to template

* better name for template

* adds checkout before other actions

* maybe

* Fixes composite

* Adds required shell for composite

* Adds name to template build

* Cleanup variables

* changing require method in typescript api

* better json parsing

* adds log

* pushing to use node 20

* Adds logs

* Updates node to 20.x

* better cancel

* Split typescript test into separate workflow

* Moves all to node 18

* Fixes package lock check

* Adds mold ?

* adds better mold

* fixes build workflow syntax

* adds log

* Better setup for mold

* fix mold path

* fix cov too

* fixes mold again

* better rustflags

* Update build.yml

* Update action.yml

* Update action.yml

* Better env vars

* Update action.yml

* better diff check

* Update action.yml

* correct variable check

* Update action.yml

* testing less parallelism in tests

* reduce more

* fixes removed timeout

* let default mocha take care of timeout

* Reduce parallel speed

* Increase timeout even more

* Force coverage to execute

* Adds wait for Eth

* Testing more parallel tests

* adds file du

* Remove useless profraws

* Forces to pass for now

* typo

* Increased timeout for default cases

* 30s timeout for ts

* Fixes provider test issues

* Restore port

* fix added logs

.github/workflow-templates/cargo-build/action.yml

@@ -0,0 +1,78 @@
+name: Cargo build
+description: |
+  Builds moonbeam with given features.
+  Stores the result in "build/moonbeam" and the runtimes in "runtimes/"
+
+inputs:
+  features:
+    description: features to include in the build (comma separated)
+    required: false
+
+env:
+  RUSTFLAGS: "-C opt-level=3 -D warnings -C linker=clang -C link-arg=-fuse-ld=./mold/bin/mold"
+  RUSTC_WRAPPER: "sccache"
+  CARGO_INCREMENTAL: "0"
+  SCCACHE_CACHE_SIZE: "100GB"
+
+runs:
+  using: "composite"
+  steps:
+    - name: Run sccache-cache
+      uses: mozilla-actions/[email protected]
+    - name: Setup Variables
+      shell: bash
+      run: |
+        echo "CARGO_INCREMENTAL=0" >> $GITHUB_ENV
+        echo "RUSTC_WRAPPER=sccache" >> $GITHUB_ENV
+        echo "SCCACHE_CACHE_SIZE=100GB" >> $GITHUB_ENV
+        # Set RUSTFLAGS if not already set
+        if [ -z "$RUSTFLAGS" ]; then
+          echo "RUSTFLAGS=-C opt-level=3 -D warnings -C linker=clang -C link-arg=-fuse-ld=$(pwd)/mold/bin/mold" >> $GITHUB_ENV
+        fi
+    - name: Setup Mold Linker
+      shell: bash
+      run: |
+        mkdir -p mold
+        curl -L --retry 10 --silent --show-error https://github.com/rui314/mold/releases/download/v1.1.1/mold-1.1.1-$(uname -m)-linux.tar.gz | tar -C $(realpath mold) --strip-components=1 -xzf -
+    # With rustup's nice new toml format, we just need to run rustup show to install the toolchain
+    # https://github.com/actions-rs/toolchain/issues/126#issuecomment-782989659
+    - name: Setup Rust toolchain
+      shell: bash
+      run: |
+        if ! which "rustup" > /dev/null; then
+          curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
+        fi
+        rustup show
+    - name: Build Node
+      shell: bash
+      run: |
+        env
+        params=" --locked --release -p moonbeam"
+        if [ -n "${{ github.event.inputs.features }}" ]; then
+          params="$params --features ${{ github.event.inputs.features }}"
+        fi
+        echo "cargo build $params"
+        cargo build $params
+    - name: Display binary comments
+      shell: bash
+      run: readelf -p .comment ./target/release/moonbeam
+    - name: Display sccache stats
+      shell: bash
+      run: ${SCCACHE_PATH} --show-stats
+    - name: Verify binary version
+      shell: bash
+      run: |
+        GIT_COMMIT=`git log -1 --format="%H" | cut -c1-7`
+        MB_VERSION=`./target/release/moonbeam --version`
+        echo "Checking $MB_VERSION contains $GIT_COMMIT"
+        echo "$MB_VERSION" | grep $GIT_COMMIT
+    - name: Save runtimes wasm
+      shell: bash
+      run: |
+        mkdir -p runtimes
+        cp target/release/wbuild/moon*/moon*_runtime.compact.compressed.wasm runtimes/;
+    - name: Save moonbeam binary
+      shell: bash
+      run: |
+        mkdir -p build
+        cp target/release/moonbeam build/moonbeam;

.github/workflow-templates/typescript-tests/action.yml

@@ -0,0 +1,107 @@
+name: Typescript tests
+description: |
+  Setup and run the typescript tests against a local moonbeam node.
+
+inputs:
+  moonbeam-binary:
+    description: path to the moonbeam binary
+    required: true
+  timeout:
+    description: default timeout for the tests
+    default: 5000
+  force-pass:
+    description: If the tests should pass even if there are failures
+    default: false
+
+env:
+  NODE_OPTIONS: "--max-old-space-size=12288"
+
+runs:
+  using: "composite"
+  steps:
+    - name: Use Node.js 18.x
+      uses: actions/setup-node@v3
+      with:
+        node-version: 18.x
+    - name: Set binary path and number of CPUs
+      shell: bash
+      run: |
+        #### Ensure the moonbeam binary is executable
+        chmod uog+x ${{ inputs.moonbeam-binary }}
+
+        #### Set the binary path (absolute path to avoid subfolder working directories issues)
+        echo "BINARY_PATH=$(realpath ${{ inputs.moonbeam-binary }})" >> $GITHUB_ENV
+
+        #### Set the number of CPUs to use for parallel tests.
+        echo "CPUS=$(lscpu | egrep '^CPU\(s\)' | grep -o '[0-9]*')" >> $GITHUB_ENV
+
+        echo "BINARY_PATH: $BINARY_PATH"
+        echo "CPUS: $CPUS"
+
+    - name: Setup Moonbeam PolkadotJS types
+      shell: bash
+      run: |
+        #### Preparing the legacy types
+        cd moonbeam-types-bundle
+        npm ci
+        npm run build
+
+        #### Preparing the typescript api
+        cd ../typescript-api
+        npm ci
+
+        cd ../tests
+        npm ci
+
+        #### Prepares and copies the typescript generated API to include in the tests
+        npm run setup-typescript-api
+
+    - name: Compile Typescript tests
+      shell: bash
+      run: |
+        #### Compile typescript tests into javascript (more stable for Mocha)
+        #### This also better display typescript issues
+        cd ./tests
+        npm run build
+
+    - name: Execute tests
+      shell: bash
+      run: |
+        cd ./tests
+        node node_modules/.bin/mocha \
+          --timeout ${{ inputs.timeout }} \
+          --parallel -j $((CPUS / 4)) \
+          --exit \
+          'build/tests/**/test-*.js' || ${{ inputs.force-pass }}
+
+    # We determine whether there are unmodified package-lock.json files by:
+    # 1. Asking git for a list of all modified files
+    # 2. Using grep to reduce the list to only package-lock.json files
+    # 3. Counting the number of lines of output
+    - name: Check package-lock.json
+      shell: bash
+      run: |
+        cd ./tests
+        # Log npm version to make sure it maches with local version
+        npm -v
+        # Make sure git is working, and if not abort early. When git is not working it looks like:
+        # $ git diff-index --name-only HEAD
+        # fatal: not a git repository (or any of the parent directories): .git
+        DIFF_INDEX="$(git diff-index --name-only HEAD)"
+        echo $DIFF_INDEX
+        if [[ -n "$DIFF_INDEX" ]]; then
+          if [[ ${DIFF_INDEX:0:5} == "fatal" ]]; then
+            echo "There was an error with the git checkout. Can't check package-lock.json file."
+            false
+          fi
+          FILECOUNT=$(echo $DIFF_INDEX | grep package-lock.json | wc -l)
+          echo $FILECOUNT
+          if [[ $FILECOUNT -eq 0 ]]; then
+            echo "All package-lock.json files are valid"
+          else
+            git diff --cached
+            echo "The following package-lock.json files have uncommitted changes"
+            echo $DIFF_INDEX | grep package-lock.json
+            false
+          fi
+        fi