fix openai model issues

Signed-off-by: SumanthRH <[email protected]>
NovaSky-AI · Feb 7, 2025 · 87ba4ab · 87ba4ab
1 parent b726e1f
commit 87ba4ab
Show file tree

Hide file tree

Showing 2 changed files with 7 additions and 6 deletions.
diff --git a/skythought/skythought_evals/inference_and_check.py b/skythought/skythought_evals/inference_and_check.py
@@ -237,7 +237,7 @@ def perform_inference_and_check(
                         results[problem_key]["messages"] = ""
                     results[problem_key]["responses"] = {}
                     results[problem_key]["token_usages"] = {}
-                    prompt = conversations[idx][1]["content"]
+                    prompt = conversations[idx][-1]["content"]
                     results[problem_key]["prompt"] = prompt
                     results[problem_key]["input_conversation"] = conversations[idx]
                     temperature_to_scores[temp][problem_key] = [
@@ -477,7 +477,7 @@ def perform_inference_and_save(
                     results[problem_key]["messages"] = ""
                 results[problem_key]["responses"] = {}
                 results[problem_key]["token_usages"] = {}
-                prompt = conversations[idx][1]["content"]
+                prompt = conversations[idx][-1]["content"]
                 results[problem_key]["prompt"] = prompt
 
             results[problem_key]["responses"][str(temp)] = response_entries

diff --git a/skythought/skythought_evals/models/model_configs.yaml b/skythought/skythought_evals/models/model_configs.yaml
@@ -21,7 +21,7 @@ system_prompts:
         <|begin_of_solution|> \
         {final formatted, precise, and clear solution} \
         <|end_of_solution|> \
-        Now, try to solve the following question through the above guidelines."
+        Now, try to solve the following question through the above guidelines:"
 
 user_templates: null
 # Example: 
@@ -32,15 +32,16 @@ models:
     # 'name' is by default <model_name> in the huggingface format <org>/<model_name>, but can be customized here
     name: o1-mini 
     system_prompt: null
-    user_template: "Question: {input}\nAnswer: "
+    # user template's use positional argument for formatting
+    user_template: "Question: {}\nAnswer: "
 
   openai/o1-preview:
     system_prompt: null
-    user_template: "Question: {input}\nAnswer: "
+    user_template: "Question: {}\nAnswer: "
 
   openai/gpt-4o-mini:
     system_prompt: null
-    user_template: "User: {input}\nAssistant: "
+    user_template: "User: {}\nAssistant: "
 
   Qwen/Qwen2-7B-Instruct:
     system_prompt: *qwen_cot_system_prompt