Generate coverage value for both unit and E2E tests (rancher#6690)

* Correct Jest coverage configuration

* Add coverage upload

* Exclude operationsl scripts from linting

* Rephrase title based on pull request title, run and attempt, with fallback for commits

* Add instrumentation for E2E coverage report

* Add merge logic to the coverage files

* Add instrumentalization for Cypress to generate coverage for E2E

* Download artifacts before merge coverage in CI

* Merge fix

* Copy coverage files before artifact upload

* Add summary after running tests

* Change restriction rules about coverage for pull_request and push

* Add documentation for the coverage

* Correct test documentation folder restructuring

* Add inclusion list for Cypress coverage; add scripts as exclusion

* Extend ignored path to be parsed for Jest to improve performances

* Workflow cleanup

* Checkout current branch head on test runs

* Submit separated coverage with different flags

.github/workflows/test.yaml

@@ -1,14 +1,19 @@
 name: Tests
 on:
+  push:
+    branches:
+      - master
+      - 'release-*'
   pull_request:
-      branches:
-        - master
-        - 'release-*'
+    branches:
+      - master
+      - 'release-*'
   workflow_dispatch:
     environment:
       description: 'Environment to run tests against'
       type: environment
       required: true 
+
 env:
   TEST_USERNAME: admin
   TEST_PASSWORD: password
@@ -17,17 +22,18 @@ env:
   API: https://127.0.0.1
   TEST_PROJECT_ID: rancher-dashboard
   CYPRESS_API_URL: http://139.59.134.103:1234/
-  TEST_RUN_ID: ${{github.event.pull_request.title}}_${{github.run_number}}_id_${{github.run_id}}
+  TEST_RUN_ID: ${{github.run_number}}-${{github.run_attempt}}-${{github.event.pull_request.title || github.event.head_commit.message}}
+  CYPRESS_coverage: true
+
 jobs:
   e2e-test:
     if: "!contains( github.event.pull_request.labels.*.name, 'ci/skip-e2e')"
     runs-on: ubuntu-latest
     steps:
-      - name: Checkout
-        uses: actions/checkout@v2
-
-      - name: Setup node
-        uses: actions/setup-node@v2
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 1
+      - uses: actions/setup-node@v3
         with:
           node-version: '14.x'
 
@@ -38,54 +44,56 @@ jobs:
         run: yarn e2e:pre-prod
 
       - name: Run tests
-        run: yarn e2e:prod
+        run: |
+          yarn e2e:prod
+          mkdir -p coverage-artifacts/coverage
+          cp coverage/e2e/coverage-final.json coverage-artifacts/coverage/coverage-e2e.json
 
+      - name: Upload coverage
+        uses: actions/upload-artifact@v3
+        with:
+          name: ${{github.run_number}}-${{github.run_attempt}}-coverage
+          path: coverage-artifacts/**/*
+
       - name: Upload screenshots
         uses: actions/upload-artifact@v2
         if: ${{ failure() }}
         with:
-          name: run_${{github.run_number}}_id_${{github.run_id}}_screenshots
+          name: ${{github.run_number}}-${{github.run_attempt}}-screenshots
           path: cypress/screenshots
 
-      # Disabled due freezing issues related to the CI machine resources
-      # - name: Upload videos
-      #   uses: actions/upload-artifact@v2
-      #   if: ${{ failure() }}
-      #   with:
-      #     name: run_${{github.run_number}}_id_${{github.run_id}}_videos
-      #     path: cypress/videos
-
   unit-test:
     runs-on: ubuntu-latest
     steps:
-      - name: Checkout
-        uses: actions/checkout@v2
-
-      - name: Setup node
-        uses: actions/setup-node@v2
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 1
+      - uses: actions/setup-node@v3
         with:
           node-version: '14.x'
 
       - name: Install packages
         run: yarn install
 
       - name: Run tests
-        run: yarn test:ci
+        run: |
+          yarn test:ci
+          mkdir -p coverage-artifacts/coverage
+          cp coverage/unit/coverage-final.json coverage-artifacts/coverage/coverage-unit.json
 
       - name: Upload coverage
         uses: actions/upload-artifact@v3
         with:
-          name: run_${{github.run_number}}_id_${{github.run_id}}_coverage
-          path: coverage/**/*
+          name: ${{github.run_number}}-${{github.run_attempt}}-coverage
+          path: coverage-artifacts/**/*
 
   i18n:
     runs-on: ubuntu-latest
     steps:
-      - name: Checkout
-        uses: actions/checkout@v2
-
-      - name: Setup node
-        uses: actions/setup-node@v2
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 1
+      - uses: actions/setup-node@v3
         with:
           node-version: '14.x'
 
@@ -98,11 +106,10 @@ jobs:
   lint:
     runs-on: ubuntu-latest
     steps:
-      - name: Checkout
-        uses: actions/checkout@v2
-
-      - name: Setup node
-        uses: actions/setup-node@v2
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 1
+      - uses: actions/setup-node@v3
         with:
           node-version: '14.x'
 
@@ -111,3 +118,51 @@ jobs:
 
       - name: Run linters
         run: yarn lint
+
+  coverage:
+    if: "!contains( github.event.pull_request.labels.*.name, 'ci/skip-e2e')"
+    runs-on: ubuntu-latest
+    needs:
+      - unit-test
+      - e2e-test
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 1
+      - uses: actions/setup-node@v3
+        with:
+          node-version: '14.x'
+
+      - name: Download Coverage Artifacts
+        uses: actions/download-artifact@v2
+        with:
+          name: ${{github.run_number}}-${{github.run_attempt}}-coverage
+
+      - name: Merge coverage files
+        run: |
+          ls
+          yarn coverage
+
+      - name: Upload unit test coverage to Codecov
+        uses: codecov/codecov-action@v3
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          flags: unit
+          files: ./coverage/coverage-unit.json
+          fail_ci_if_error: true
+
+      - name: Upload e2e test coverage to Codecov
+        uses: codecov/codecov-action@v3
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          flags: e2e
+          files: ./coverage/coverage-e2e.json
+          fail_ci_if_error: true
+
+      - name: Upload merged coverage to Codecov
+        uses: codecov/codecov-action@v3
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          flags: merged
+          files: ./coverage/coverage.json
+          fail_ci_if_error: true

.nycrc

@@ -0,0 +1,4 @@
+{
+  "reporter": [ "json", "text-summary"],
+  "report-dir": "coverage/e2e"
+}

.vscode/settings.json

@@ -19,6 +19,7 @@
     "autoscroll",
     "cacerts",
     "chainable",
+    "Codecov",
     "epinio",
     "hevi",
     "kube",

cypress.config.ts

@@ -1,4 +1,5 @@
 import { defineConfig } from 'cypress';
+// Required for env vars to be available in cypress
 require('dotenv').config();
 
 /**
@@ -23,20 +24,33 @@ const getSpecPattern = (): string[] => {
 const baseUrl = (process.env.TEST_BASE_URL || 'https://localhost:8005').replace(/\/$/, '');
 
 export default defineConfig({
-  projectId:             process.env.TEST_PROJECT_ID,
-  defaultCommandTimeout: 60000,
-  trashAssetsBeforeRuns: true,
-  env:                   {
+  projectId:              process.env.TEST_PROJECT_ID,
+  defaultCommandTimeout:  60000,
+  trashAssetsBeforeRuns:  true,
+  env:                    {
     baseUrl,
-    username:          process.env.TEST_USERNAME,
-    password:          process.env.TEST_PASSWORD,
-    bootstrapPassword: process.env.CATTLE_BOOTSTRAP_PASSWORD,
+    coverage:             false,
+    codeCoverage: {
+      exclude: [
+        'cypress/**/*.*',
+        '**/__tests__/**/*.*',
+        '**/shell/scripts/**/*.*',
+      ],
+      include: [
+        'shell/**/*.{vue,ts,js}',
+        'pkg/rancher-components/src/components/**/*.{vue,ts,js}',
+      ]
+    },
+    username:             process.env.TEST_USERNAME,
+    password:             process.env.TEST_PASSWORD,
+    bootstrapPassword:    process.env.CATTLE_BOOTSTRAP_PASSWORD,
   },
   e2e:                   {
-    // We've imported your old cypress plugins here.
-    // You may want to clean this up later by importing these.
     setupNodeEvents(on, config) {
-      return require('./cypress/plugins/index.ts')(on, config);
+      // For more info: https://docs.cypress.io/guides/tooling/code-coverage
+      require('@cypress/code-coverage/task')(on, config);
+
+      return config;
     },
     experimentalSessionAndOrigin: true,
     specPattern:                  getSpecPattern(),

cypress/globals.d.ts

@@ -0,0 +1,38 @@
+type Matcher = '$' | '^' | '~' | '*' | '';
+
+// eslint-disable-next-line no-unused-vars
+declare namespace Cypress {
+  interface Chainable {
+    login(username?: string, password?: string, cacheSession?: boolean): Chainable<Element>;
+    byLabel(label: string,): Chainable<Element>;
+
+    /**
+     *  Wrapper for cy.get() to simply define the data-testid value that allows you to pass a matcher to find the element.
+     * @param id Value used for the data-testid attribute of the element.
+     * @param matcher Matching character used for attribute value:
+     * - `$`: Suffixed with this value
+     * - `^`: Prefixed with this value
+     * - `~`: Contains this value as whitespace separated words
+     * - `*`: Contains this value
+     */
+    getId(id: string, matcher?: Matcher): Chainable<Element>;
+
+    /**
+     *  Wrapper for cy.find() to simply define the data-testid value that allows you to pass a matcher to find the element.
+     * @param id Value used for the data-testid attribute of the element.
+     * @param matcher Matching character used for attribute value:
+     * - `$`: Suffixed with this value
+     * - `^`: Prefixed with this value
+     * - `~`: Contains this value as whitespace separated words
+     * - `*`: Contains this value
+     */
+    findId(id: string, matcher?: Matcher): Chainable<Element>;
+
+    /**
+     * Override user preferences to default values, allowing to pass custom preferences for a deterministic scenario
+     * Leave empty for reset to default values
+     */
+    // eslint-disable-next-line no-undef
+    userPreferences(preferences?: Partial<UserPreferences>): Chainable<null>;
+  }
+}