Adds code coverage to CI (#2273)

* Adds coverage CI * testing full coverage * increase timeout * removes default timeout in code * Increase even more * force passing tests * better cp of prof files * fix coverage path * better success :) * support invalid coverage file :( * adds rm to free up space after coverage run * add llvm-cov * fix file name * Better comments * adding dev mode to the ci * better check ? * debug ci * good ref * remoev dev mode * dev mode fgalse * display dev mode * better dev mode * better true/false * typo * pfff :( * checking error * debugging missing files * restore invalid check * Testing to upload coverage * typo * Report coverage info * anothe test * better coverage * Enable full coverage * Adds cargo test * Restore coverage artifact * Removes target build from coverage * testing different * Compare coverage with master * Ignore build release files * generate master folder in S3 * restore pulls branches * force master * restore non master * retry master * restore pr * fixes coverage to master * restore to pulls * Better build filtering * Adds cleanup folder * testing better ignore * restore build * restore test * Makes variable more generic
moonbeam-foundation · May 11, 2023 · 6e2c6c7 · 6e2c6c7
1 parent 6f61568
commit 6e2c6c7
Showing 1 changed file with 263 additions and 0 deletions.
diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml
@@ -0,0 +1,263 @@
+name: Coverage
+
+# Using a single file workflow is the preferred solution for our CI over workflow_runs.
+# 1. It generates only 1 action item in the list making it more readable
+# 2. It includes the PR/Commit text in the action item
+# 3. Artifacts are not available between workflows.
+
+on:
+  pull_request:
+  push:
+    branches:
+      - master
+env:
+  SCCACHE_CACHE_SIZE: "100GB"
+  CARGO_INCREMENTAL: "0"
+
+jobs:
+  ####### Check files and formatting #######
+
+  set-tags:
+    runs-on: ubuntu-latest
+    outputs:
+      git_branch: ${{ steps.check-git-ref.outputs.git_branch }}
+      git_ref: ${{ steps.check-git-ref.outputs.git_ref }}
+      image_exists: ${{ steps.check-docker-image.outputs.image_exists }}
+      sha: ${{ steps.get-sha.outputs.sha }}
+      sha8: ${{ steps.get-sha.outputs.sha8 }}
+      polkadot_repo: ${{ steps.get-sha.outputs.polkadot_repo }}
+      polkadot_commit: ${{ steps.get-sha.outputs.polkadot_commit }}
+      latest_rt: ${{ steps.get-sha.outputs.latest_rt }}
+      latest_rt_sha8: ${{ steps.get-sha.outputs.latest_rt_sha8 }}
+      coverage_dir: ${{ steps.check-git-ref.outputs.coverage_dir }}
+      coverage_report: ${{ steps.check-git-ref.outputs.coverage_report }}
+    steps:
+      - name: Check git ref
+        id: check-git-ref
+        # if PR
+        # else if manual PR
+        # else (push)
+        run: |
+          if [[ -n "${{ github.event.pull_request.head.sha }}" ]]; then
+            echo "git_branch=$(echo ${GITHUB_HEAD_REF})" >> $GITHUB_OUTPUT
+            echo "git_ref=${{ github.event.pull_request.head.sha }}" >> $GITHUB_OUTPUT
+            echo "coverage_dir=pulls/${{ github.event.pull_request.number }}" >> $GITHUB_OUTPUT
+            echo "coverage_report=true" >> $GITHUB_OUTPUT
+          else
+            echo "git_branch=$(echo ${GITHUB_REF#refs/heads/})" >> $GITHUB_OUTPUT
+            echo "git_ref=$GITHUB_REF" >> $GITHUB_OUTPUT
+            echo "coverage_dir=branches/master" >> $GITHUB_OUTPUT
+            echo "coverage_report=false" >> $GITHUB_OUTPUT
+          fi
+
+      - uses: actions/checkout@v3
+        with:
+          ref: ${{ steps.check-git-ref.outputs.git_ref }}
+      - name: Get Latest RT Release
+        id: get-latest-rt
+        run: |
+          LATEST_RUNTIME_RELEASE=$(curl -s https://api.github.com/repos/purestake/moonbeam/releases | jq -r '.[] | select(.name | test("runtime";"i")) | .tag_name' | head -n 1 | tr -d '[:blank:]')
+          echo $LATEST_RUNTIME_RELEASE
+          echo "latest_rt=$LATEST_RUNTIME_RELEASE" >> $GITHUB_OUTPUT
+      - name: Get Sha
+        id: get-sha
+        run: |
+          echo "sha=$(git log -1 --format='%H')" >> $GITHUB_OUTPUT
+          echo "sha8=$(git log -1 --format='%H' | cut -c1-8)" >> $GITHUB_OUTPUT
+          echo "polkadot_repo=$(egrep -o 'https.*/polkadot' Cargo.lock | head -1)" >> $GITHUB_OUTPUT
+          echo "polkadot_commit=$(egrep -o '/polkadot.*#([^\"]*)' Cargo.lock | \
+          head -1 | sed 's/.*#//' |  cut -c1-8)" >> $GITHUB_OUTPUT
+
+          echo "$(curl -s "https://api.github.com/repos/purestake/moonbeam/git/refs/tags/${{ steps.get-latest-rt.outputs.latest_rt }}" | jq -r '.object.sha' | cut -c 1-8)"
+          echo "latest_rt_sha8=$(curl -s 'https://api.github.com/repos/purestake/moonbeam/git/refs/tags/${{steps.get-latest-rt.outputs.latest_rt }}' | jq -r '.object.sha' | cut -c 1-8 )" >> $GITHUB_OUTPUT
+      - name: Check existing docker image
+        id: check-docker-image
+        run: |
+          TAG=sha-${{ steps.get-sha.outputs.sha8 }}
+          echo "image_exists=$(docker manifest inspect purestake/moonbeam:$TAG > /dev/null && echo "true" || echo "false")" >> $GITHUB_OUTPUT
+      - name: Display variables
+        run: |
+          echo git_ref: ${{ steps.check-git-ref.outputs.git_ref }}
+          echo sha: ${{ steps.get-sha.outputs.sha }}
+          echo sha8: ${{ steps.get-sha.outputs.sha8 }}
+          echo image_exists: ${{ steps.check-docker-image.outputs.image_exists }}
+          echo latest_rt: ${{ steps.get-latest-rt.outputs.latest_rt }}
+          echo latest_rt_sha8: ${{ steps.get-sha.outputs.latest_rt_sha8 }}
+
+  build-and-coverage:
+    runs-on:
+      labels: bare-metal
+    needs: ["set-tags"]
+    env:
+      RUSTFLAGS: "-C opt-level=3 -D warnings -C instrument-coverage"
+      RUSTC_WRAPPER: "sccache"
+      CARGO_TARGET_DIR: "target"
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          ref: ${{ needs.set-tags.outputs.git_ref }}
+      - name: Run sccache-cache
+        uses: mozilla-actions/[email protected]
+      # With rustup's nice new toml format, we just need to run rustup show to install the toolchain
+      # https://github.com/actions-rs/toolchain/issues/126#issuecomment-782989659
+      - name: Setup Rust toolchain
+        run: |
+          if ! which "rustup" > /dev/null; then
+            curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
+          fi
+          rustup show
+      - name: Setup grcov
+        run: |
+          wget https://alan-stuff.s3.amazonaws.com/grcov -O grcov
+          chmod +x grcov
+      - name: Setup llvmprof-data
+        run: |
+          wget https://alan-stuff.s3.amazonaws.com/llvm-profdata -O llvm-profdata
+          wget https://alan-stuff.s3.amazonaws.com/llvm-cov -O llvm-cov 
+          chmod +x llvm-profdata
+          chmod +x llvm-cov
+      - name: Build Node
+        run: |
+          env
+          cargo build --locked --release -p moonbeam
+      - name: Run sccache stat for check pre test
+        shell: bash
+        run: ${SCCACHE_PATH} --show-stats
+      - name: Verify node version
+        run: |
+          GIT_COMMIT=`git log -1 --format="%H" | cut -c1-7`
+          MB_VERSION=`./$CARGO_TARGET_DIR/release/moonbeam --version`
+          echo "Checking $MB_VERSION contains $GIT_COMMIT"
+          echo "$MB_VERSION" | grep $GIT_COMMIT
+      - name: Save runtime wasm
+        run: |
+          mkdir -p runtimes
+          cp $CARGO_TARGET_DIR/release/wbuild/moon*/moon*_runtime.compact.compressed.wasm runtimes/;
+      - name: Upload runtimes
+        uses: actions/[email protected]
+        with:
+          name: runtimes
+          path: runtimes
+      - name: Save moonbeam binary
+        run: |
+          mkdir -p build
+          cp $CARGO_TARGET_DIR/release/moonbeam build/moonbeam;
+      - name: Upload binary
+        uses: actions/[email protected]
+        with:
+          name: moonbeam
+          path: build
+      - name: Unit tests
+        run: |
+          cargo test --release --all --features=evm-tracing
+      - name: Use Node.js 20.x
+        uses: actions/setup-node@v3
+        with:
+          node-version: 20.x
+      - name: Typescript integration tests (against dev service)
+        run: |
+          chmod uog+x build/moonbeam
+
+          ####  Preparing the repository
+          cd moonbeam-types-bundle
+          npm ci
+          npm run build
+
+          ####  Preparing the typescript api
+          cd ../typescript-api
+          npm ci
+
+          cd ../tests
+          npm ci
+          #### Prepares and copies the typescript generated API to include in the tests
+          npm run setup-typescript-api 
+
+          #### Compile typescript tests into javascript (more stable for Mocha)
+          #### This also better display typescript issues
+          npm run build
+
+          CPUS=$(lscpu | egrep '^CPU\(s\)' | grep -o '[0-9]*')
+
+          node node_modules/.bin/mocha \
+            --timeout 180000 \
+            --parallel -j $((CPUS / 2)) \
+            --exit \
+            'build/tests/**/test-*.js'  || true
+          # We force it to pass to get the coverage report
+      - name: Retrieve coverage
+        id: coverage
+        run: |
+          mkdir -p /tmp/proffiles
+
+          echo "Deletes invalid raw files (<74Mb) not suze why we have them"
+          find . -type f -name \*.profraw -exec ls -l {}  \;
+
+          echo "Copying profraw files to /tmp/proffiles"
+          find . -name \*.profraw -exec mv {} /tmp/proffiles/ \;
+
+          mv /tmp/proffiles proffiles
+
+          echo "Executing grcov"
+          ./grcov proffiles/ -s ./ --binary-path ./${CARGO_TARGET_DIR}/release/ \
+            -t html --branch --ignore-not-existing --ignore ${CARGO_TARGET_DIR}/release/build/* \
+            -o coverage/ --llvm-path ./  --llvm 2>&1 \
+            | tee grcov.log
+
+          INVALID="$(grep invalid grcov.log | \
+            grep -o '[^\ /]*raw' | \
+            sed 's/_1\.profraw/\.profraw/g' || echo '')"
+          echo "Invalid files: $INVALID"
+          if [ -n "$INVALID" ]; then
+            echo "Removing invalid files: $INVALID"
+            cd proffiles/
+            rm $INVALID
+            cd ..
+            ./grcov proffiles/ -s ./ --binary-path ./${CARGO_TARGET_DIR}/release/ \
+            -t html --branch --ignore-not-existing --ignore ${CARGO_TARGET_DIR}/release/build/* \
+            -o coverage/ --llvm-path ./  --llvm
+          fi
+
+          echo "coverage_date=\"$(date)\"" >> $GITHUB_OUTPUT
+          echo "total_percent=$(grep -o '[0-9\.]*%' coverage/html/coverage.json)" >> $GITHUB_OUTPUT
+          wget https://${{ vars.S3_COVERAGE_BUCKET }}.s3.amazonaws.com/branches/master/html/coverage.json \
+            -O coverage-master.json || true
+          echo "master_percent=$(grep -o '[0-9\.]*%' coverage/html/coverage-master.json || echo 'N/A')" >> $GITHUB_OUTPUT
+          rm -rf proffiles/
+      - name: Upload coverate to gha
+        uses: actions/[email protected]
+        with:
+          name: coverage
+          path: coverage
+      - name: Upload coverage s3
+        uses: shallwefootball/s3-upload-action@master
+        id: S3
+        with:
+          aws_key_id: ${{ secrets.S3_COVERAGE_ID }}
+          aws_secret_access_key: ${{ secrets.S3_COVERAGE_KEY }}
+          aws_bucket: ${{ vars.S3_COVERAGE_BUCKET }}
+          destination_dir: "${{ needs.set-tags.outputs.coverage_dir }}"
+          source_dir: "coverage"
+      - name: Link To Report
+        run: echo "https://s3.amazonaws.com/${{ vars.S3_COVERAGE_BUCKET }}/${{steps.S3.outputs.object_key}}/html/index.html"
+      - name: Find Comment
+        if: ${{ needs.set-tags.outputs.coverage_report == 'true' }}
+        uses: peter-evans/find-comment@v2
+        id: fc
+        with:
+          issue-number: ${{ github.event.pull_request.number }}
+          comment-author: "github-actions[bot]"
+          body-includes: Coverage generated
+      - name: Create or update comment
+        if: ${{ needs.set-tags.outputs.coverage_report == 'true' }}
+        uses: peter-evans/create-or-update-comment@v3
+        with:
+          comment-id: ${{ steps.fc.outputs.comment-id }}
+          issue-number: ${{ github.event.pull_request.number }}
+          body: |
+            Coverage generated ${{ steps.coverage.outputs.coverage_date }}:
+            https://s3.amazonaws.com/${{ vars.S3_COVERAGE_BUCKET }}/${{steps.S3.outputs.object_key}}/html/index.html
+
+            Master coverage: ${{ steps.coverage.outputs.master_percent }}
+            Pull coverage: ${{ steps.coverage.outputs.total_percent }}
+          edit-mode: replace