Skip to content

Commit

Permalink
Added first LLMCompare version
Browse files Browse the repository at this point in the history
  • Loading branch information
josebastos committed Oct 14, 2023
1 parent b87c0a4 commit 96ee13d
Show file tree
Hide file tree
Showing 3 changed files with 160 additions and 17 deletions.
13 changes: 3 additions & 10 deletions api/logs/execution_logs.jsonl
View file
Original file line number Diff line number Diff line change
@@ -1,10 +1,3 @@
{"id": 394, "chatInput": "Hi", "chatOutput": "Hello! How can I assist you today?", "inputTokens": 1, "outputTokens": 9, "totalTokens": 10, "cost": 0.00057, "timestamp": 1696582868.3290176, "modelName": "gpt-4", "parameters": {"temperature": 1.0, "max_tokens": 256, "top_p": 1.0, "frequency_penalty": 0.0, "presence_penalty": 0.0}}
{"id": 292, "chatInput": "hi", "chatOutput": " \n", "inputTokens": 2, "outputTokens": 2, "totalTokens": 4, "cost": 2e-06, "timestamp": 1696582937.6345363, "modelName": "chat-bison", "parameters": {"temperature": 1.0, "max_tokens": 256, "top_p": 1.0, "top_k": 40.0}}
{"id": 598, "chatInput": "Give me ten interview questions for the role of program manager.", "chatOutput": " 1. **What experience do you have in managing programs?**\n2. **What are your strengths and weaknesses as a program manager?**\n3. **What is your approach to managing a program?**\n4. **What tools and techniques do you use to manage programs?**\n5. **How do you manage risks and issues in a program?**\n6. **How do you communicate with stakeholders in a program?**\n7. **How do you manage changes in a program?**\n8. **How do you measure success in a program?**\n9. **What is your approach to continuous improvement in a program?**\n10. **What is your biggest failure as a program manager?**", "inputTokens": 64, "outputTokens": 582, "totalTokens": 646, "cost": 0.000323, "timestamp": 1696604237.3852518, "modelName": "text-bison", "parameters": {"temperature": 1, "max_tokens": 256, "top_p": 1, "top_k": 40}}
{"id": 903, "chatInput": "How many planets are there in the solar system?", "chatOutput": " There are eight planets orbiting our sun\n", "inputTokens": 47, "outputTokens": 42, "totalTokens": 89, "cost": 4.45e-05, "timestamp": 1696604255.0318148, "modelName": "chat-bison", "parameters": {"temperature": 1, "max_tokens": 256, "top_p": 1, "top_k": 40}}
{"id": 913, "chatInput": "Write a function that checks if a year is a leap year.", "chatOutput": "```python\ndef is_leap_year(year):\n \"\"\"\n Checks if a year is a leap year.\n\n Args:\n year: The year to check.\n\n Returns:\n True if the year is a leap year, False otherwise.\n \"\"\"\n\n if year % 4 != 0:\n return False\n\n if year % 100 == 0 and year % 400 != 0:\n return False\n\n return True\n```", "inputTokens": 54, "outputTokens": 301, "totalTokens": 355, "cost": 0.0001775, "timestamp": 1696604723.3980658, "modelName": "code-bison", "parameters": {"temperature": 1, "max_tokens": 256, "top_p": 1, "top_k": 40}}
{"id": 63, "chatInput": "Please help write a function to calculate the min of two numbers.", "chatOutput": " ```python\ndef min_of_two_numbers(a, b):\n \"\"\"Returns the minimum of two numbers.\"\"\"\n if a < b:\n return a\n else:\n return b\n```", "inputTokens": 65, "outputTokens": 134, "totalTokens": 199, "cost": 9.949999999999999e-05, "timestamp": 1696604754.1339984, "modelName": "codechat-bison", "parameters": {"temperature": 1, "max_tokens": 256, "top_p": 1, "top_k": 40}}
{"id": 19, "chatInput": "hey", "chatOutput": " Hey! How can I help you?", "inputTokens": 3, "outputTokens": 25, "totalTokens": 28, "cost": 1.4e-05, "timestamp": 1696605659.5679395, "modelName": "text-bison", "parameters": {"temperature": 1.0, "max_tokens": 256, "top_p": 1.0, "top_k": 40.0}}
{"id": 217, "chatInput": "How many planets are there in the solar system?", "chatOutput": " Eight: Mercury, Venus, Earth, Mars, Jupiter, Saturn, Uranus, and Neptune\n", "inputTokens": 47, "outputTokens": 74, "totalTokens": 121, "cost": 6.05e-05, "timestamp": 1696605689.0663517, "modelName": "chat-bison", "parameters": {"temperature": 1.0, "max_tokens": 256, "top_p": 1.0, "top_k": 40.0}}
{"id": 216, "chatInput": "Write a function that checks if a year is a leap year.", "chatOutput": "```python\ndef is_leap_year(year):\n \"\"\"\n Checks if a year is a leap year.\n\n Args:\n year: The year to check.\n\n Returns:\n True if the year is a leap year, False otherwise.\n \"\"\"\n\n if year % 4 != 0:\n return False\n\n if year % 100 == 0 and year % 400 != 0:\n return False\n\n return True\n```", "inputTokens": 54, "outputTokens": 301, "totalTokens": 355, "cost": 0.0001775, "timestamp": 1696605701.2161443, "modelName": "code-bison", "parameters": {"temperature": 1.0, "max_tokens": 256, "top_p": 1.0, "top_k": 40.0}}
{"id": 882, "chatInput": "Please help write a function to calculate the min of two numbers", "chatOutput": " ```python\ndef min_of_two_numbers(a, b):\n \"\"\"Returns the minimum of two numbers.\"\"\"\n if a < b:\n return a\n else:\n return b\n```", "inputTokens": 64, "outputTokens": 134, "totalTokens": 198, "cost": 9.9e-05, "timestamp": 1696605714.5077055, "modelName": "codechat-bison", "parameters": {"temperature": 1.0, "max_tokens": 256, "top_p": 1.0, "top_k": 40.0}}
{"id": 904, "chatInput": "Hello, how are you?", "chatOutput": "As an artificial intelligence, I don't have feelings, but I'm functioning as expected. Thank you! How can I assist you today?", "inputTokens": 6, "outputTokens": 28, "totalTokens": 34, "cost": 0.00186, "timestamp": 1697198275.8091068, "modelName": "gpt-4", "parameters": {"temperature": 1.0, "max_tokens": 256, "top_p": 1.0, "frequency_penalty": 0.0, "presence_penalty": 0.0}}
{"id": 34, "chatInput": "Hello, my name is Claudio. I am a Data Scientist.", "chatOutput": "Hello Claudio! Nice to meet you. How can I assist you today as a Data Scientist?", "inputTokens": 14, "outputTokens": 20, "totalTokens": 34, "cost": 6.1e-05, "timestamp": 1697275769.368771, "modelName": "gpt-3.5-turbo", "parameters": {"temperature": 1.0, "max_tokens": 256, "top_p": 1.0, "frequency_penalty": 0.0, "presence_penalty": 0.0}}
{"id": 476, "chatInput": "Hello, my name is Claudio. I am a Data Scientist.", "chatOutput": "Hello Claudio! It's interesting to meet a Data Scientist. How can I assist you in your profession today?", "inputTokens": 14, "outputTokens": 23, "totalTokens": 37, "cost": 0.0018, "timestamp": 1697275771.5668528, "modelName": "gpt-4", "parameters": {"temperature": 1.0, "max_tokens": 256, "top_p": 1.0, "frequency_penalty": 0.0, "presence_penalty": 0.0}}
124 changes: 124 additions & 0 deletions draft.ipynb
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,124 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"id": "initial_id",
"metadata": {
"ExecuteTime": {
"end_time": "2023-10-13T11:45:46.937179Z",
"start_time": "2023-10-13T11:45:46.931633Z"
},
"collapsed": true
},
"outputs": [],
"source": [
"from llmstudio.models import OpenAIClient, VertexAIClient"
]
},
{
"cell_type": "code",
"execution_count": 2,
"id": "e806accc8ba79ff5",
"metadata": {
"ExecuteTime": {
"end_time": "2023-10-13T11:45:47.123938Z",
"start_time": "2023-10-13T11:45:47.119305Z"
},
"collapsed": false
},
"outputs": [],
"source": [
"client= OpenAIClient(api_key='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')\n",
"\n"
]
},
{
"cell_type": "code",
"execution_count": 3,
"id": "afc629a014268524",
"metadata": {
"ExecuteTime": {
"end_time": "2023-10-13T11:45:59.331432Z",
"start_time": "2023-10-13T11:45:59.110621Z"
},
"collapsed": false
},
"outputs": [],
"source": [
"model = client.get_model(model_name='gpt-4')\n"
]
},
{
"cell_type": "code",
"execution_count": 4,
"id": "5539a85865d09fa6",
"metadata": {
"ExecuteTime": {
"end_time": "2023-10-13T11:46:05.558922Z",
"start_time": "2023-10-13T11:46:05.525759Z"
},
"collapsed": false
},
"outputs": [
{
"data": {
"text/plain": [
"{'id': 904,\n",
" 'chatInput': 'Hello, how are you?',\n",
" 'chatOutput': \"As an artificial intelligence, I don't have feelings, but I'm functioning as expected. Thank you! How can I assist you today?\",\n",
" 'inputTokens': 6,\n",
" 'outputTokens': 28,\n",
" 'totalTokens': 34,\n",
" 'cost': 0.00186,\n",
" 'timestamp': 1697198275.8091068,\n",
" 'modelName': 'gpt-4',\n",
" 'parameters': {'temperature': 1.0,\n",
" 'max_tokens': 256,\n",
" 'top_p': 1.0,\n",
" 'frequency_penalty': 0.0,\n",
" 'presence_penalty': 0.0}}"
]
},
"execution_count": 4,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"model.chat('Hello, how are you?')"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "43796b32ecf8550b",
"metadata": {
"collapsed": false
},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.9.13"
}
},
"nbformat": 4,
"nbformat_minor": 5
}
40 changes: 33 additions & 7 deletions llmstudio/models/models.py
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
from abc import ABC, abstractmethod
import requests
from pydantic import BaseModel
import threading


class LLMModel(ABC):
Expand All @@ -25,11 +26,11 @@ class LLMModel(ABC):

@abstractmethod
def __init__(
self,
model_name: str,
api_key: str = None,
api_secret: str = None,
api_region: str = None,
self,
model_name: str,
api_key: str = None,
api_secret: str = None,
api_region: str = None,
):
"""
Initialize the LLMModel instance.
Expand Down Expand Up @@ -65,7 +66,7 @@ def _check_api_access(self):
raise ValueError(
f"The API key doesn't have access to {self.model_name}"
)

@abstractmethod
def validate_parameters(self, parameters: BaseModel) -> BaseModel:
"""
Expand Down Expand Up @@ -142,7 +143,7 @@ class LLMClient(ABC):
MODEL_MAPPING = {}

def __init__(
self, api_key: str = None, api_secret: str = None, api_region: str = None
self, api_key: str = None, api_secret: str = None, api_region: str = None
):
"""
Initialize the LLMClient instance.
Expand Down Expand Up @@ -182,3 +183,28 @@ def get_model(self, model_name: str):
api_secret=self.api_secret,
api_region=self.api_region,
)


class LLMCompare(ABC):

def __int__(self):
pass

def _get_response_from_model(self, model, prompt, output_dict):
output_dict[model.model_name] = model.chat(prompt)
return output_dict

def compare(self, models, prompt: str, ):
threads = []
output_dict = {}

for model in models:
thread = threading.Thread(target=self._get_response_from_model, args=(model, prompt, output_dict))
thread.start()
threads.append(thread)

for thread in threads:
thread.join()

return output_dict

0 comments on commit 96ee13d

Please sign in to comment.