rework llm based detection #226

server/application-server/src/main/java/de/tum/in/www1/hephaestus/activity/badpracticedetector/PullRequestBadPracticeDetector.java

@@ -1,12 +1,8 @@
 package de.tum.in.www1.hephaestus.activity.badpracticedetector;
 
 import de.tum.in.www1.hephaestus.activity.PullRequestBadPracticeRepository;
-import de.tum.in.www1.hephaestus.activity.PullRequestBadPracticeRuleRepository;
-import de.tum.in.www1.hephaestus.activity.model.BadPracticeType;
 import de.tum.in.www1.hephaestus.activity.model.PullRequestBadPractice;
-import de.tum.in.www1.hephaestus.activity.model.PullRequestBadPracticeRule;
 import de.tum.in.www1.hephaestus.gitprovider.pullrequest.PullRequest;
-
 import java.util.LinkedList;
 import java.util.List;
 import java.util.stream.Collectors;
@@ -33,19 +29,14 @@ public class PullRequestBadPracticeDetector {
     @Autowired
     private PullRequestBadPracticeRepository pullRequestBadPracticeRepository;
 
-    @Autowired
-    private PullRequestBadPracticeRuleRepository pullRequestBadPracticeRuleRepository;
-
     private final DetectorApi detectorApi = new DetectorApi();
 
     public List<PullRequestBadPractice> detectAndSyncBadPractices(List<PullRequest> pullRequests) {
         logger.info("Detecting bad practices for pull requests.");
 
-        List<PullRequestBadPracticeRule> rules = pullRequestBadPracticeRuleRepository.findAllActive();
 
         DetectorRequest detectorRequest = new DetectorRequest();
         detectorRequest.setPullRequests(mapToApiPullRequests(pullRequests));
-        detectorRequest.setRules(mapToApiRules(rules));
         DetectorResponse detectorResponse = detectorApi.detectDetectorPost(detectorRequest);
 
         List<PullRequestBadPractice> detectedBadPractices = new LinkedList<>();
@@ -75,7 +66,6 @@ private List<PullRequestBadPractice> handleDetectedBadPractices(PullRequestWithB
                 .map(badPracticeRule -> {
                     PullRequestBadPractice pullRequestBadPractice = new PullRequestBadPractice();
                     pullRequestBadPractice.setPullrequest(pullRequest);
-                    pullRequestBadPractice.setType(findBadPracticeType(badPracticeRule));
                     pullRequestBadPractice.setResolved(false);
                     return pullRequestBadPractice;
                 })

server/intelligence-service/app/detector/bad_practice_detector.py

@@ -1,28 +1,33 @@
 from typing import List
 
-from pydantic import BaseModel
+from langchain_core.prompts import PromptTemplate, ChatPromptTemplate
+from pydantic import BaseModel, Field
+
+from ..model import model
 
 
 class PullRequest(BaseModel):
     id: str
     title: str
     description: str
 
-class Rule(BaseModel):
-    name: str
-    description: str
-    bad_practice_id: str
-
-class PullRequestWithBadPractices(BaseModel):
-    pull_request_id: str
-    bad_practice_ids: List[str]
-
-def detectbadpractices(pull_requests: List[PullRequest], rules: List[Rule]) -> List[PullRequestWithBadPractices]:
-    bad_practices = []
-    for pull_request in pull_requests:
-        bad_practice_ids = []
-        for rule in rules:
-            if rule.bad_practice_id in pull_request.description:
-                bad_practice_ids.append(rule.bad_practice_id)
-        bad_practices.append(PullRequestWithBadPractices(pull_request_id=pull_request.id, bad_practice_ids=bad_practice_ids))
-    return bad_practices
+class BadPractice(BaseModel):
+    """A detected bad practice in a pull request."""
+
+    title: str = Field(description="The title of the bad practice.")
+    description: str = Field(description="The description of the bad practice.")
+
+class BadPracticeList(BaseModel):
+    """A list of bad practices detected in a pull request."""
+
+    bad_practices: List[BadPractice] = Field(description="A list of bad practices detected in a pull request.")
+
+
+def detectbadpractices(pull_requests: PullRequest) -> BadPracticeList:
+   with open("./prompts/pullrequest_badpractice_detector.txt", "r") as f:
+       prompt_text = f.read()
+   prompt_template = ChatPromptTemplate.from_template(prompt_text)
+   prompt = prompt_template.invoke(title=pull_requests.title, description=pull_requests.description)
+   structured_llm = model.with_structured_output(BadPracticeList)
+   response = structured_llm.invoke(prompt)
+   return response

server/intelligence-service/app/detector/prompts/pullrequest_badpractice_detector

@@ -0,0 +1,28 @@
+You are a bad practice detector reviewing pull requests for bad practices.
+You analyze and review the title and description of the pull request to identify any bad practices.
+You detect bad practices based on guidelines for good pull request titles and descriptions.
+
+PRIMARY TASK:
+Detect and identify any bad practices in the provided pull request title and description.
+- Review the title and description for any issues or violations of the guidelines.
+- For each detected bad practice, provide a title and a brief description of the issue.
+- Return a list of all detected bad practices in the pull request.
+
+GUIDELINES:
+1. The title and description should be clear, concise, and descriptive.
+2. The title should be specific and summarize the changes made in the pull request.
+3. The description should provide additional context and details about the changes.
+4. The title and description should be free of spelling and grammatical errors.
+5. The title and description should follow the project's guidelines and conventions.
+6. The description should not contain empty sections or placeholder text.
+7. The description should not include unchecked checkboxes or unresolved action items.
+8. The description should not include open TODOs
+9. In the description the motivation and description sections should be filled out.
+10. The description should not be empty.
+
+REQUIREMENTS:
+1. Identify and describe all bad practices in the pull request title and description.
+2. Provide a clear title and a brief and concise description for each detected bad practice.
+
+Pull Request Title: {title}
+Pull Request Description: {description}

server/intelligence-service/app/routers/detector.py

@@ -1,21 +1,12 @@
-from typing import List
 from fastapi import APIRouter
-from pydantic import BaseModel
-from ..detector.bad_practice_detector import PullRequestWithBadPractices, PullRequest, Rule, detectbadpractices
+from ..detector.bad_practice_detector import PullRequest, detectbadpractices, BadPracticeList
 
 router = APIRouter(prefix="/detector", tags=["detector"])
 
-class DetectorRequest(BaseModel):
-    pull_requests: List[PullRequest]
-    rules: List[Rule]
-
-class DetectorResponse(BaseModel):
-    detectBadPractices: List[PullRequestWithBadPractices]
-
 @router.post(
     "/",
-    response_model=DetectorResponse,
+    response_model=BadPracticeList,
     summary="Detect bad practices given rules.",
 )
-def detect(request: DetectorRequest):
-    return detectbadpractices(request.pull_requests, request.rules)
+def detect(request: PullRequest):
+    return detectbadpractices(request)