Introduce Nightly Regression Pipeline (#100)

* Updated tests, dockerfile, and created new nightly workflow. * Enabling manual trigger on branch for testing. * Fixed depends for GLIBC. * Triggering single test. * Trigger on branch. * Moving to bash. * Adding bash. * Running in container. * Adding installer build. * Adding debug. * Adding debug 2. * Moving to Ubuntu 22.04. * Downloading from main's release binary. * Incrementing version. * Building installer and then running integration tests. * Adding execution permissions to cover-agent. * Fixing OpenAI token and cache. * Enabling all tests. * Adding cache and moving to Python version 3.12. * Moving regrassion testing pipeline to nightly. * Adding Docs. Resolves #99 #84 #65 #38
qodo-ai · Jun 17, 2024 · 619b023 · 619b023
1 parent 94d9996
commit 619b023
Show file tree

Hide file tree

Showing 9 changed files with 212 additions and 65 deletions.
diff --git a/.github/workflows/ci_pipeline.yml b/.github/workflows/ci_pipeline.yml
@@ -38,7 +38,19 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v2
         with:
-          python-version: '3.x'
+          python-version: '3.12'
+
+      # Step to cache Poetry dependencies
+      - name: Cache Poetry dependencies
+        uses: actions/cache@v2
+        with:
+          path: |
+            ~/.cache/pypoetry
+            ~/.cache/pip
+          key: ${{ runner.os }}-poetry-${{ hashFiles('**/poetry.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-poetry-
+
       - name: Install Poetry
         run: pip install poetry
       - name: Install dependencies using Poetry
@@ -71,7 +83,19 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v2
         with:
-          python-version: '3.x'
+          python-version: '3.12'
+
+      # Step to cache Poetry dependencies
+      - name: Cache Poetry dependencies
+        uses: actions/cache@v2
+        with:
+          path: |
+            ~/.cache/pypoetry
+            ~/.cache/pip
+          key: ${{ runner.os }}-poetry-${{ hashFiles('**/poetry.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-poetry-
+
       - name: Install Poetry
         run: pip install poetry
       - name: Install dependencies using Poetry
@@ -88,7 +112,7 @@ jobs:
     if: needs.changes.outputs.relevant_changes == 'true'
     strategy:
       matrix:
-       os: [ubuntu-latest, windows-latest, macos-latest]
+       os: [ubuntu-22.04, windows-latest, macos-latest]
 
     runs-on: ${{ matrix.os }}
     steps:
@@ -97,6 +121,18 @@ jobs:
       uses: actions/setup-python@v2
       with:
         python-version: '3.12'
+
+    # Step to cache Poetry dependencies
+    - name: Cache Poetry dependencies
+      uses: actions/cache@v2
+      with:
+        path: |
+          ~/.cache/pypoetry
+          ~/.cache/pip
+        key: ${{ runner.os }}-poetry-${{ hashFiles('**/poetry.lock') }}
+        restore-keys: |
+          ${{ runner.os }}-poetry-
+
     - name: Install Dependencies
       run: |
         pip install poetry wandb
@@ -130,8 +166,8 @@ jobs:
     - name: Download executables (Ubuntu)
       uses: actions/download-artifact@v2
       with:
-        name: cover-agent-ubuntu-latest
-        path: dist/ubuntu-latest
+        name: cover-agent-ubuntu-22.04
+        path: dist/ubuntu-22.04
     - uses: actions/download-artifact@v2
       with:
         name: cover-agent-windows-latest
@@ -159,7 +195,7 @@ jobs:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       with:
         upload_url: ${{ steps.create_release.outputs.upload_url }}
-        asset_path: dist/ubuntu-latest/cover-agent
+        asset_path: dist/ubuntu-22.04/cover-agent
         asset_name: cover-agent-ubuntu
         asset_content_type: application/octet-stream
     - name: Upload Release Asset (Windows)

diff --git a/.github/workflows/nightly_regression.yml b/.github/workflows/nightly_regression.yml
@@ -0,0 +1,76 @@
+name: Nightly Integration Tests
+
+# Define the events that will trigger this workflow
+on:
+  # Schedule the workflow to run nightly at midnight
+  schedule:
+    - cron: '0 0 * * *'
+  # Allow manual triggering of the workflow
+  workflow_dispatch:
+
+jobs:
+  build-executable:
+    runs-on: ubuntu-22.04
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: '3.12'
+
+    # Step to cache Poetry dependencies
+    - name: Cache Poetry dependencies
+      uses: actions/cache@v2
+      with:
+        path: |
+          ~/.cache/pypoetry
+          ~/.cache/pip
+        key: ${{ runner.os }}-poetry-${{ hashFiles('**/poetry.lock') }}
+        restore-keys: |
+          ${{ runner.os }}-poetry-
+
+    - name: Install Dependencies
+      run: |
+        pip install poetry wandb
+        poetry install
+    - name: Build Executable
+      run: make installer
+    - name: Upload Executable
+      uses: actions/upload-artifact@v2
+      with:
+        name: cover-agent
+        path: dist/cover-agent*
+
+  run-integration-tests:
+    needs: build-executable
+    runs-on: ubuntu-latest
+    container:
+      image: docker:latest
+      options: --privileged # Required for Docker-in-Docker
+
+    env:
+      OPENAI_API_KEY: ${{ secrets.OPENAI_KEY }}
+
+    steps:
+    # Step 1: Check out the repository code
+    - name: Checkout repository
+      uses: actions/checkout@v2
+
+    # Step 2: Download the artifact
+    - name: Download artifact
+      uses: actions/download-artifact@v2
+      with:
+        name: cover-agent
+        path: dist/
+
+    # Step 3: Run integration tests
+    - name: Run integration tests
+      run: |
+        # Set permissions for dist/cover-agent
+        chmod +x dist/cover-agent
+
+        # Install bash
+        apk add --no-cache bash
+        
+        # Execute the shell script with bash
+        bash tests_integration/test_all.sh
diff --git a/Dockerfile b/Dockerfile
@@ -1,5 +1,5 @@
 # Dockerfile to build installer for Cover Agent
-FROM python:3.12
+FROM python:3.12-bullseye
 
 WORKDIR /app
 

diff --git a/cover_agent/version.txt b/cover_agent/version.txt
@@ -1 +1 @@
-0.1.42
+0.1.43
diff --git a/templated_tests/java_spring_calculator/Dockerfile b/templated_tests/java_spring_calculator/Dockerfile
@@ -3,9 +3,13 @@ FROM maven:latest
 # Set the working directory inside the container
 WORKDIR /app
 
+# Install glibc
+RUN apt-get update && \
+    apt-get install -y libc6
+
 # Copy the pom.xml file and the source code
 COPY pom.xml .
 COPY src ./src
 
 # Package the application and run tests
-RUN mvn clean verify
+RUN mvn clean verify
diff --git a/..._calculator/src/test/java/com/example/calculator/controller/CalculatorControllerTest.java b/..._calculator/src/test/java/com/example/calculator/controller/CalculatorControllerTest.java
@@ -45,23 +45,4 @@ public void testSubtract() throws Exception {
                 .andExpect(content().string("1.0"));
     }
 
-    @Test
-    public void testMultiply() throws Exception {
-        mockMvc.perform(get("/multiply?a=2&b=3"))
-                .andExpect(status().isOk())
-                .andExpect(content().string("6.0"));
-    }
-
-    @Test
-    public void testDivide() throws Exception {
-        mockMvc.perform(get("/divide?a=6&b=3"))
-                .andExpect(status().isOk())
-                .andExpect(content().string("2.0"));
-    }
-
-    // @Test
-    // public void testDivideByZero() throws Exception {
-    //     mockMvc.perform(get("/divide?a=1&b=0"))
-    //             .andExpect(status().isBadRequest());
-    // }
 }
diff --git a/templated_tests/js_vanilla/Dockerfile b/templated_tests/js_vanilla/Dockerfile
@@ -0,0 +1,12 @@
+FROM node:18
+
+WORKDIR /app
+
+# Install glibc
+RUN apt-get update && \
+    apt-get install -y libc6
+
+COPY . ./
+
+RUN npm install
+RUN npm run test:coverage
diff --git a/tests_integration/README.md b/tests_integration/README.md
@@ -1,62 +1,97 @@
-# Integration tests for Cover Agent
-This folder contains end-to-end integration tests for Cover Agent
+# Integration Tests for Cover Agent
+This folder contains end-to-end integration tests for Cover Agent.
 
 ## Prerequisites
-Before running any of these tests you will need to build the installer package by running the following command from the root of the repository:
+Before running any of these tests, you will need to build the installer package by running the following command from the root of the repository:
 ```
 make installer
 ```
 
 You will also need [Docker](https://www.docker.com/) installed.
 
-__Note:__ These scripts were written for Linux but they have been tested on a Windows system using WSL 2 and Docker for Desktop.
+__Note:__ These scripts were written for Linux but have been tested on a Windows system using WSL 2 and Docker for Desktop.
 
-Since the targets live in Linux you'll need to build the installer in Linux (versus on Windows and MacOS). This can be done automatically in the `sh tests_integration/test_all.sh` script by add the `--run-installer` flag.
+Since the targets live in Linux, you'll need to build the installer in Linux (versus on Windows and MacOS). This can be done automatically in the `sh tests_integration/test_all.sh` script by adding the `--run-installer` flag.
 
-## How to run
-You can run these example test suites using a locally hosted LLM or in the cloud just as you would normally with Cover Agent
+## How to Run
+You can run these example test suites using a locally hosted LLM or in the cloud just as you would normally with Cover Agent.
 
 ### Running the Tests
-To run the full test suite simply run the following command from the root of the repository:
+To run the full test suite, simply run the following command from the root of the repository:
 ```
 sh tests_integration/test_all.sh
 ```
 
 Or run each test individually:
 #### Python Fast API Example
-````
+```
 sh tests_integration/test_with_docker.sh \
   --dockerfile "templated_tests/python_fastapi/Dockerfile" \
   --source-file-path "app.py" \
   --test-file-path "test_app.py" \
   --test-command "pytest --cov=. --cov-report=xml --cov-report=term"
-````
+```
 
 #### Go Webservice Example
-````
+```
 sh tests_integration/test_with_docker.sh \
   --dockerfile "templated_tests/go_webservice/Dockerfile" \
   --source-file-path "app.go" \
   --test-file-path "app_test.go" \
   --test-command "go test -coverprofile=coverage.out && gocov convert coverage.out | gocov-xml > coverage.xml" \
   --model "gpt-4o"
-````
+```
+
+#### Java Gradle Example
+```
+sh tests_integration/test_with_docker.sh \
+  --dockerfile "templated_tests/java_gradle/Dockerfile" \
+  --source-file-path "src/main/java/com/davidparry/cover/SimpleMathOperations.java" \
+  --test-file-path "src/test/groovy/com/davidparry/cover/SimpleMathOperationsSpec.groovy" \
+  --test-command "./gradlew clean test jacocoTestReport" \
+  --coverage-type "jacoco" \
+  --code-coverage-report-path "build/reports/jacoco/test/jacocoTestReport.csv" \
+  --model "gpt-4o"
+```
+
+#### Java Spring Calculator Example
+```
+sh tests_integration/test_with_docker.sh \
+  --dockerfile "templated_tests/java_spring_calculator/Dockerfile" \
+  --source-file-path "src/main/java/com/example/calculator/controller/CalculatorController.java" \
+  --test-file-path "src/test/java/com/example/calculator/controller/CalculatorControllerTest.java" \
+  --test-command "mvn verify" \
+  --coverage-type "jacoco" \
+  --code-coverage-report-path "target/site/jacoco/jacoco.csv" \
+  --model "gpt-4o"
+```
+
+#### VanillaJS Example
+```
+sh tests_integration/test_with_docker.sh \
+  --dockerfile "templated_tests/js_vanilla/Dockerfile" \
+  --source-file-path "ui.js" \
+  --test-file-path "ui.test.js" \
+  --test-command "npm run test:coverage" \
+  --code-coverage-report-path "coverage/coverage.xml" \
+  --model "gpt-4o"
+```
 
 ### Using Different LLMs
 You can use a different LLM by passing in the `--model` and `--api-base` parameters. For example, to use a locally hosted LLM with Ollama you can pass in:
 ```
 --model "ollama/mistral" --api-base "http://host.docker.internal:11434"
 ```
 
-For any other LLM that requires more environment variables to be set you will need to update the shell script and pass in the variables within the Docker command.
+For any other LLM that requires more environment variables to be set, you will need to update the shell script and pass in the variables within the Docker command.
 
-## When to run
-This test suite is intended to run with real LLMs (either locally hosted on online). If choosing cloud provided LLMs keep in mind that there is a cost associated with running these tests.
+## When to Run
+This test suite is intended to run with real LLMs (either locally hosted or online). If choosing cloud-provided LLMs, keep in mind that there is a cost associated with running these tests.
 
 These tests should **absolutely** be run before a massive refactor or any major changes to the LLM/prompting logic of the code.
 
-## How it works
-The integration tests run within Docker containers in order to prove out complete isolation from any external or existing environment
+## How It Works
+The integration tests run within Docker containers to ensure complete isolation from any external or existing environment.
 
 # Increasing Coverage Iteratively
 The `increase_coverage.py` script attempts to run Cover Agent for all files within the `cover_agent` directory. You'll need to call a Poetry shell first before running like so: