diff --git a/.github/workflows/ci_pipeline.yml b/.github/workflows/ci_pipeline.yml index 9bc0cfc37..cee958952 100644 --- a/.github/workflows/ci_pipeline.yml +++ b/.github/workflows/ci_pipeline.yml @@ -38,7 +38,19 @@ jobs: - name: Set up Python uses: actions/setup-python@v2 with: - python-version: '3.x' + python-version: '3.12' + + # Step to cache Poetry dependencies + - name: Cache Poetry dependencies + uses: actions/cache@v2 + with: + path: | + ~/.cache/pypoetry + ~/.cache/pip + key: ${{ runner.os }}-poetry-${{ hashFiles('**/poetry.lock') }} + restore-keys: | + ${{ runner.os }}-poetry- + - name: Install Poetry run: pip install poetry - name: Install dependencies using Poetry @@ -71,7 +83,19 @@ jobs: - name: Set up Python uses: actions/setup-python@v2 with: - python-version: '3.x' + python-version: '3.12' + + # Step to cache Poetry dependencies + - name: Cache Poetry dependencies + uses: actions/cache@v2 + with: + path: | + ~/.cache/pypoetry + ~/.cache/pip + key: ${{ runner.os }}-poetry-${{ hashFiles('**/poetry.lock') }} + restore-keys: | + ${{ runner.os }}-poetry- + - name: Install Poetry run: pip install poetry - name: Install dependencies using Poetry @@ -88,7 +112,7 @@ jobs: if: needs.changes.outputs.relevant_changes == 'true' strategy: matrix: - os: [ubuntu-latest, windows-latest, macos-latest] + os: [ubuntu-22.04, windows-latest, macos-latest] runs-on: ${{ matrix.os }} steps: @@ -97,6 +121,18 @@ jobs: uses: actions/setup-python@v2 with: python-version: '3.12' + + # Step to cache Poetry dependencies + - name: Cache Poetry dependencies + uses: actions/cache@v2 + with: + path: | + ~/.cache/pypoetry + ~/.cache/pip + key: ${{ runner.os }}-poetry-${{ hashFiles('**/poetry.lock') }} + restore-keys: | + ${{ runner.os }}-poetry- + - name: Install Dependencies run: | pip install poetry wandb @@ -130,8 +166,8 @@ jobs: - name: Download executables (Ubuntu) uses: actions/download-artifact@v2 with: - name: cover-agent-ubuntu-latest - path: dist/ubuntu-latest + name: cover-agent-ubuntu-22.04 + path: dist/ubuntu-22.04 - uses: actions/download-artifact@v2 with: name: cover-agent-windows-latest @@ -159,7 +195,7 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: upload_url: ${{ steps.create_release.outputs.upload_url }} - asset_path: dist/ubuntu-latest/cover-agent + asset_path: dist/ubuntu-22.04/cover-agent asset_name: cover-agent-ubuntu asset_content_type: application/octet-stream - name: Upload Release Asset (Windows) diff --git a/.github/workflows/nightly_regression.yml b/.github/workflows/nightly_regression.yml new file mode 100644 index 000000000..99c2170b0 --- /dev/null +++ b/.github/workflows/nightly_regression.yml @@ -0,0 +1,76 @@ +name: Nightly Integration Tests + +# Define the events that will trigger this workflow +on: + # Schedule the workflow to run nightly at midnight + schedule: + - cron: '0 0 * * *' + # Allow manual triggering of the workflow + workflow_dispatch: + +jobs: + build-executable: + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v2 + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: '3.12' + + # Step to cache Poetry dependencies + - name: Cache Poetry dependencies + uses: actions/cache@v2 + with: + path: | + ~/.cache/pypoetry + ~/.cache/pip + key: ${{ runner.os }}-poetry-${{ hashFiles('**/poetry.lock') }} + restore-keys: | + ${{ runner.os }}-poetry- + + - name: Install Dependencies + run: | + pip install poetry wandb + poetry install + - name: Build Executable + run: make installer + - name: Upload Executable + uses: actions/upload-artifact@v2 + with: + name: cover-agent + path: dist/cover-agent* + + run-integration-tests: + needs: build-executable + runs-on: ubuntu-latest + container: + image: docker:latest + options: --privileged # Required for Docker-in-Docker + + env: + OPENAI_API_KEY: ${{ secrets.OPENAI_KEY }} + + steps: + # Step 1: Check out the repository code + - name: Checkout repository + uses: actions/checkout@v2 + + # Step 2: Download the artifact + - name: Download artifact + uses: actions/download-artifact@v2 + with: + name: cover-agent + path: dist/ + + # Step 3: Run integration tests + - name: Run integration tests + run: | + # Set permissions for dist/cover-agent + chmod +x dist/cover-agent + + # Install bash + apk add --no-cache bash + + # Execute the shell script with bash + bash tests_integration/test_all.sh diff --git a/Dockerfile b/Dockerfile index 504654c9f..d991715ca 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # Dockerfile to build installer for Cover Agent -FROM python:3.12 +FROM python:3.12-bullseye WORKDIR /app diff --git a/cover_agent/version.txt b/cover_agent/version.txt index 9c30d51df..2fbba850f 100644 --- a/cover_agent/version.txt +++ b/cover_agent/version.txt @@ -1 +1 @@ -0.1.42 \ No newline at end of file +0.1.43 \ No newline at end of file diff --git a/templated_tests/java_spring_calculator/Dockerfile b/templated_tests/java_spring_calculator/Dockerfile index 2a6d608e1..ba59fff08 100644 --- a/templated_tests/java_spring_calculator/Dockerfile +++ b/templated_tests/java_spring_calculator/Dockerfile @@ -3,9 +3,13 @@ FROM maven:latest # Set the working directory inside the container WORKDIR /app +# Install glibc +RUN apt-get update && \ + apt-get install -y libc6 + # Copy the pom.xml file and the source code COPY pom.xml . COPY src ./src # Package the application and run tests -RUN mvn clean verify \ No newline at end of file +RUN mvn clean verify diff --git a/templated_tests/java_spring_calculator/src/test/java/com/example/calculator/controller/CalculatorControllerTest.java b/templated_tests/java_spring_calculator/src/test/java/com/example/calculator/controller/CalculatorControllerTest.java index 97aa90e32..55fb64dee 100644 --- a/templated_tests/java_spring_calculator/src/test/java/com/example/calculator/controller/CalculatorControllerTest.java +++ b/templated_tests/java_spring_calculator/src/test/java/com/example/calculator/controller/CalculatorControllerTest.java @@ -45,23 +45,4 @@ public void testSubtract() throws Exception { .andExpect(content().string("1.0")); } - @Test - public void testMultiply() throws Exception { - mockMvc.perform(get("/multiply?a=2&b=3")) - .andExpect(status().isOk()) - .andExpect(content().string("6.0")); - } - - @Test - public void testDivide() throws Exception { - mockMvc.perform(get("/divide?a=6&b=3")) - .andExpect(status().isOk()) - .andExpect(content().string("2.0")); - } - - // @Test - // public void testDivideByZero() throws Exception { - // mockMvc.perform(get("/divide?a=1&b=0")) - // .andExpect(status().isBadRequest()); - // } } diff --git a/templated_tests/js_vanilla/Dockerfile b/templated_tests/js_vanilla/Dockerfile new file mode 100644 index 000000000..fd6945f69 --- /dev/null +++ b/templated_tests/js_vanilla/Dockerfile @@ -0,0 +1,12 @@ +FROM node:18 + +WORKDIR /app + +# Install glibc +RUN apt-get update && \ + apt-get install -y libc6 + +COPY . ./ + +RUN npm install +RUN npm run test:coverage diff --git a/tests_integration/README.md b/tests_integration/README.md index bada6e3bc..ecffb9fc8 100644 --- a/tests_integration/README.md +++ b/tests_integration/README.md @@ -1,46 +1,81 @@ -# Integration tests for Cover Agent -This folder contains end-to-end integration tests for Cover Agent +# Integration Tests for Cover Agent +This folder contains end-to-end integration tests for Cover Agent. ## Prerequisites -Before running any of these tests you will need to build the installer package by running the following command from the root of the repository: +Before running any of these tests, you will need to build the installer package by running the following command from the root of the repository: ``` make installer ``` You will also need [Docker](https://www.docker.com/) installed. -__Note:__ These scripts were written for Linux but they have been tested on a Windows system using WSL 2 and Docker for Desktop. +__Note:__ These scripts were written for Linux but have been tested on a Windows system using WSL 2 and Docker for Desktop. -Since the targets live in Linux you'll need to build the installer in Linux (versus on Windows and MacOS). This can be done automatically in the `sh tests_integration/test_all.sh` script by add the `--run-installer` flag. +Since the targets live in Linux, you'll need to build the installer in Linux (versus on Windows and MacOS). This can be done automatically in the `sh tests_integration/test_all.sh` script by adding the `--run-installer` flag. -## How to run -You can run these example test suites using a locally hosted LLM or in the cloud just as you would normally with Cover Agent +## How to Run +You can run these example test suites using a locally hosted LLM or in the cloud just as you would normally with Cover Agent. ### Running the Tests -To run the full test suite simply run the following command from the root of the repository: +To run the full test suite, simply run the following command from the root of the repository: ``` sh tests_integration/test_all.sh ``` Or run each test individually: #### Python Fast API Example -```` +``` sh tests_integration/test_with_docker.sh \ --dockerfile "templated_tests/python_fastapi/Dockerfile" \ --source-file-path "app.py" \ --test-file-path "test_app.py" \ --test-command "pytest --cov=. --cov-report=xml --cov-report=term" -```` +``` #### Go Webservice Example -```` +``` sh tests_integration/test_with_docker.sh \ --dockerfile "templated_tests/go_webservice/Dockerfile" \ --source-file-path "app.go" \ --test-file-path "app_test.go" \ --test-command "go test -coverprofile=coverage.out && gocov convert coverage.out | gocov-xml > coverage.xml" \ --model "gpt-4o" -```` +``` + +#### Java Gradle Example +``` +sh tests_integration/test_with_docker.sh \ + --dockerfile "templated_tests/java_gradle/Dockerfile" \ + --source-file-path "src/main/java/com/davidparry/cover/SimpleMathOperations.java" \ + --test-file-path "src/test/groovy/com/davidparry/cover/SimpleMathOperationsSpec.groovy" \ + --test-command "./gradlew clean test jacocoTestReport" \ + --coverage-type "jacoco" \ + --code-coverage-report-path "build/reports/jacoco/test/jacocoTestReport.csv" \ + --model "gpt-4o" +``` + +#### Java Spring Calculator Example +``` +sh tests_integration/test_with_docker.sh \ + --dockerfile "templated_tests/java_spring_calculator/Dockerfile" \ + --source-file-path "src/main/java/com/example/calculator/controller/CalculatorController.java" \ + --test-file-path "src/test/java/com/example/calculator/controller/CalculatorControllerTest.java" \ + --test-command "mvn verify" \ + --coverage-type "jacoco" \ + --code-coverage-report-path "target/site/jacoco/jacoco.csv" \ + --model "gpt-4o" +``` + +#### VanillaJS Example +``` +sh tests_integration/test_with_docker.sh \ + --dockerfile "templated_tests/js_vanilla/Dockerfile" \ + --source-file-path "ui.js" \ + --test-file-path "ui.test.js" \ + --test-command "npm run test:coverage" \ + --code-coverage-report-path "coverage/coverage.xml" \ + --model "gpt-4o" +``` ### Using Different LLMs You can use a different LLM by passing in the `--model` and `--api-base` parameters. For example, to use a locally hosted LLM with Ollama you can pass in: @@ -48,15 +83,15 @@ You can use a different LLM by passing in the `--model` and `--api-base` paramet --model "ollama/mistral" --api-base "http://host.docker.internal:11434" ``` -For any other LLM that requires more environment variables to be set you will need to update the shell script and pass in the variables within the Docker command. +For any other LLM that requires more environment variables to be set, you will need to update the shell script and pass in the variables within the Docker command. -## When to run -This test suite is intended to run with real LLMs (either locally hosted on online). If choosing cloud provided LLMs keep in mind that there is a cost associated with running these tests. +## When to Run +This test suite is intended to run with real LLMs (either locally hosted or online). If choosing cloud-provided LLMs, keep in mind that there is a cost associated with running these tests. These tests should **absolutely** be run before a massive refactor or any major changes to the LLM/prompting logic of the code. -## How it works -The integration tests run within Docker containers in order to prove out complete isolation from any external or existing environment +## How It Works +The integration tests run within Docker containers to ensure complete isolation from any external or existing environment. # Increasing Coverage Iteratively The `increase_coverage.py` script attempts to run Cover Agent for all files within the `cover_agent` directory. You'll need to call a Poetry shell first before running like so: diff --git a/tests_integration/test_all.sh b/tests_integration/test_all.sh index 2d144f92f..649a49615 100755 --- a/tests_integration/test_all.sh +++ b/tests_integration/test_all.sh @@ -5,7 +5,7 @@ set -o pipefail # Exit if any command in a pipeline fails set -x # Print commands and their arguments as they are executed # Default model name -MODEL="gpt-3.5-turbo" +MODEL="gpt-4o" RUN_INSTALLER=false # Function to display usage @@ -35,16 +35,12 @@ done # Conditional Docker commands if [ "$RUN_INSTALLER" = true ]; then - # Get the current user ID and group ID - USER_ID=$(id -u) - GROUP_ID=$(id -g) - # Build the installer within a Docker container docker build -t cover-agent-installer -f Dockerfile . # Run the Docker container with the current user's ID and group ID mkdir -p dist - docker run --rm --user $USER_ID:$GROUP_ID --volume "$(pwd)/dist:/app/dist" cover-agent-installer + docker run --rm --volume "$(pwd)/dist:/app/dist" cover-agent-installer fi # Python FastAPI Example @@ -53,19 +49,17 @@ sh tests_integration/test_with_docker.sh \ --source-file-path "app.py" \ --test-file-path "test_app.py" \ --test-command "pytest --cov=. --cov-report=xml --cov-report=term" \ - --model $MODEL + --model "gpt-3.5-turbo" # Go Webservice Example -# Note: GPT 3.5 hasn't been powerful enough to generate quality tests that pass sh tests_integration/test_with_docker.sh \ --dockerfile "templated_tests/go_webservice/Dockerfile" \ --source-file-path "app.go" \ --test-file-path "app_test.go" \ --test-command "go test -coverprofile=coverage.out && gocov convert coverage.out | gocov-xml > coverage.xml" \ - --model "gpt-4o" + --model $MODEL # Java Gradle example -# Note: GPT 3.5 hasn't been powerful enough to generate quality tests that pass sh tests_integration/test_with_docker.sh \ --dockerfile "templated_tests/java_gradle/Dockerfile" \ --source-file-path "src/main/java/com/davidparry/cover/SimpleMathOperations.java" \ @@ -73,14 +67,23 @@ sh tests_integration/test_with_docker.sh \ --test-command "./gradlew clean test jacocoTestReport" \ --coverage-type "jacoco" \ --code-coverage-report-path "build/reports/jacoco/test/jacocoTestReport.csv" \ - --model "gpt-4o" + --model $MODEL -# # Java Spring Calculator example -# sh tests_integration/test_with_docker.sh \ -# --dockerfile "templated_tests/java_spring_calculator/Dockerfile" \ -# --source-file-path "src/main/java/com/example/calculator/controller/CalculatorController.java" \ -# --test-file-path "src/test/java/com/example/calculator/controller/CalculatorControllerTest.java" \ -# --test-command "mvn verify" \ -# --coverage-type "jacoco" \ -# --code-coverage-report-path "target/site/jacoco/jacoco.xml" \ -# --model $MODEL +# Java Spring Calculator example +sh tests_integration/test_with_docker.sh \ + --dockerfile "templated_tests/java_spring_calculator/Dockerfile" \ + --source-file-path "src/main/java/com/example/calculator/controller/CalculatorController.java" \ + --test-file-path "src/test/java/com/example/calculator/controller/CalculatorControllerTest.java" \ + --test-command "mvn verify" \ + --coverage-type "jacoco" \ + --code-coverage-report-path "target/site/jacoco/jacoco.csv" \ + --model $MODEL + +# VanillaJS Example +sh tests_integration/test_with_docker.sh \ + --dockerfile "templated_tests/js_vanilla/Dockerfile" \ + --source-file-path "ui.js" \ + --test-file-path "ui.test.js" \ + --test-command "npm run test:coverage" \ + --code-coverage-report-path "coverage/coverage.xml" \ + --model $MODEL \ No newline at end of file