Use structured output with JSON schema, when supported

ariya · Nov 2, 2024 · f09b463 · f09b463
1 parent e25e1c5
commit f09b463
Show file tree

Hide file tree

Showing 4 changed files with 232 additions and 61 deletions.
diff --git a/.github/workflows/test-gemini.yml b/.github/workflows/test-gemini.yml
@@ -30,6 +30,7 @@ jobs:
           LLM_API_BASE_URL: 'https://generativelanguage.googleapis.com/v1beta'
           LLM_API_KEY: ${{ secrets.GEMINI_API_KEY }}
           LLM_CHAT_MODEL: 'gemini-1.5-flash-8b'
+          LLM_JSON_SCHEMA: 1
 
       - run: cat output.txt
       - run: grep -i jupiter output.txt
@@ -46,6 +47,7 @@ jobs:
           LLM_API_BASE_URL: 'https://generativelanguage.googleapis.com/v1beta'
           LLM_API_KEY: ${{ secrets.GEMINI_API_KEY }}
           LLM_CHAT_MODEL: 'gemini-1.5-flash-8b'
+          LLM_JSON_SCHEMA: 1
 
   high-school-stem:
     needs: chain-of-thought
@@ -59,6 +61,7 @@ jobs:
           LLM_API_BASE_URL: 'https://generativelanguage.googleapis.com/v1beta'
           LLM_API_KEY: ${{ secrets.GEMINI_API_KEY }}
           LLM_CHAT_MODEL: 'gemini-1.5-flash-8b'
+          LLM_JSON_SCHEMA: 1
 
   general-knowledge:
     needs: chain-of-thought
@@ -72,3 +75,4 @@ jobs:
           LLM_API_BASE_URL: 'https://generativelanguage.googleapis.com/v1beta'
           LLM_API_KEY: ${{ secrets.GEMINI_API_KEY }}
           LLM_CHAT_MODEL: 'gemini-1.5-flash-8b'
+          LLM_JSON_SCHEMA: 1
diff --git a/.github/workflows/test-gpt.yml b/.github/workflows/test-gpt.yml
@@ -30,6 +30,8 @@ jobs:
             LLM_API_BASE_URL: 'https://api.openai.com/v1'
             LLM_API_KEY: ${{ secrets.OPENAI_API_KEY }}
             LLM_CHAT_MODEL: 'gpt-4o-mini'
+            LLM_JSON_SCHEMA: 1
+            LLM_DEBUG_CHAT: 1
 
       - run: cat output.txt
       - run: grep -i jupiter output.txt
@@ -46,6 +48,8 @@ jobs:
             LLM_API_BASE_URL: 'https://api.openai.com/v1'
             LLM_API_KEY: ${{ secrets.OPENAI_API_KEY }}
             LLM_CHAT_MODEL: 'gpt-4o-mini'
+            LLM_JSON_SCHEMA: 1
+            LLM_DEBUG_CHAT: 1
 
   high-school-stem:
     needs: chain-of-thought
@@ -59,6 +63,8 @@ jobs:
             LLM_API_BASE_URL: 'https://api.openai.com/v1'
             LLM_API_KEY: ${{ secrets.OPENAI_API_KEY }}
             LLM_CHAT_MODEL: 'gpt-4o-mini'
+            LLM_JSON_SCHEMA: 1
+            LLM_DEBUG_CHAT: 1
 
   general-knowledge:
     needs: chain-of-thought
@@ -72,3 +78,4 @@ jobs:
             LLM_API_BASE_URL: 'https://api.openai.com/v1'
             LLM_API_KEY: ${{ secrets.OPENAI_API_KEY }}
             LLM_CHAT_MODEL: 'gpt-4o-mini'
+            LLM_JSON_SCHEMA: 1
diff --git a/.github/workflows/test-small-llm.yml b/.github/workflows/test-small-llm.yml
@@ -42,6 +42,7 @@ jobs:
       - run: echo 'Which planet in our solar system is the largest?' | ./query-llm.js | tee output.txt
         env:
           LLM_API_BASE_URL: 'http://127.0.0.1:8080/v1'
+          LLM_JSON_SCHEMA: 1
 
       - run: cat output.txt
       - run: grep -i jupiter output.txt
@@ -64,6 +65,7 @@ jobs:
       - run: ./query-llm.js tests/canary-multi-turn.txt
         env:
           LLM_API_BASE_URL: 'http://127.0.0.1:8080/v1'
+          LLM_JSON_SCHEMA: 1
 
   high-school-stem:
     needs: chain-of-thought
@@ -83,3 +85,4 @@ jobs:
       - run: ./query-llm.js tests/high-school-stem.txt
         env:
           LLM_API_BASE_URL: 'http://127.0.0.1:8080/v1'
+          LLM_JSON_SCHEMA: 1