From c61aae0467e3a166809f065aeae44541a7ce8e1c Mon Sep 17 00:00:00 2001 From: Maddy Adams Date: Tue, 9 Jul 2024 17:46:03 -0700 Subject: [PATCH 01/61] feat: move hub sdk functionality to langsmith sdk --- python/langsmith/client.py | 175 +++++++++++++++++++++++++++++++++++ python/langsmith/schemas.py | 38 ++++++++ python/langsmith/utils.py | 25 +++++ python/tests/hub/__init__.py | 0 4 files changed, 238 insertions(+) create mode 100644 python/tests/hub/__init__.py diff --git a/python/langsmith/client.py b/python/langsmith/client.py index bd39b0e5a..e67ec8e21 100644 --- a/python/langsmith/client.py +++ b/python/langsmith/client.py @@ -4561,6 +4561,181 @@ def _evaluate_strings( ) + def get_settings(self): + res = requests.get( + f"{self.api_url}/settings", + headers=self._headers, + ) + res.raise_for_status() + return res.json() + + + def list_prompts(self, limit: int = 100, offset: int = 0) -> ls_schemas.ListPromptsResponse: + res = requests.get( + f"{self.api_url}/repos?limit={limit}&offset={offset}", + headers=self._headers, + ) + res.raise_for_status() + res_dict = res.json() + return ls_schemas.ListPromptsResponse(**res_dict) + + + def get_prompt(self, prompt_identifier: str) -> ls_schemas.Prompt: + res = requests.get( + f"{self.api_url}/repos/{prompt_identifier}", + headers=self._headers, + ) + res.raise_for_status() + prompt = res.json()['repo'] + return ls_schemas.Prompt(**prompt) + + + def create_prompt( + self, prompt_name: str, *, description: str = "", is_public: bool = True + ): + json = { + "repo_handle": prompt_name, + "is_public": is_public, + "description": description, + } + res = requests.post( + f"{self.api_url}/repos/", + headers=self._headers, + json=json, + ) + res.raise_for_status() + return res.json() + + + def list_commits(self, prompt_name: str, limit: int = 100, offset: int = 0): + res = requests.get( + f"{self.api_url}/commits/{prompt_name}/?limit={limit}&offset={offset}", + headers=self._headers, + ) + res.raise_for_status() + return res.json() + + + def _get_latest_commit_hash(self, prompt_identifier: str) -> Optional[str]: + commits_resp = self.list_commits(prompt_identifier) + commits = commits_resp["commits"] + if len(commits) == 0: + return None + return commits[0]["commit_hash"] + + + def pull_prompt( + self, + prompt_identifier: str, + ) -> ls_schemas.PromptManifest: + """Pull a prompt from the LangSmith API. + + Args: + prompt_identifier: The identifier of the prompt (str, ex. "prompt_name", "owner/prompt_name", "owner/prompt_name:commit_hash") + + Yields: + Prompt + The prompt + """ + LS_VERSION_WITH_OPTIMIZATION="0.5.23" + use_optimization = ls_utils.is_version_greater_or_equal( + current_version=self.info.version, target_version=LS_VERSION_WITH_OPTIMIZATION + ) + + owner, prompt_name, commit_hash = ls_utils.parse_prompt_identifier(prompt_identifier) + + if not use_optimization: + if commit_hash is None or commit_hash == "latest": + commit_hash = self._get_latest_commit_hash(f"{owner}/{prompt_name}") + if commit_hash is None: + raise ValueError("No commits found") + + res = requests.get( + f"{self.api_url}/commits/{owner}/{prompt_name}/{commit_hash}", + headers=self._headers, + ) + res.raise_for_status() + result = res.json() + return ls_schemas.PromptManifest(**{"owner": owner, "repo": prompt_name, **result}) + + def push_prompt( + self, + prompt_identifier: str, + manifest_json: Any, + *, + parent_commit_hash: Optional[str] = "latest", + is_public: bool = False, + description: str = "", + ) -> str: + """Push a prompt to the LangSmith API. + + Args: + prompt_name: The name of the prompt + manifest_json: The JSON string of the prompt manifest + parent_commit_hash: The commit hash of the parent commit + is_public: Whether the new prompt is public + description: The description of the new prompt + """ + from langchain_core.load.dump import dumps + manifest_json = dumps(manifest_json) + settings = self.get_settings() + if is_public: + if not settings["tenant_handle"]: + raise ValueError( + """ + Cannot create public prompt without first creating a LangChain Hub handle. + + You can add a handle by creating a public prompt at: + https://smith.langchain.com/prompts + + This is a workspace-level handle and will be associated with all of your workspace's public prompts in the LangChain Hub. + """ + ) + owner, prompt_name, _ = ls_utils.parse_prompt_identifier(prompt_identifier) + prompt_full_name = f"{owner}/{prompt_name}" + try: + # check if the prompt exists + _ = self.get_prompt(prompt_full_name) + except requests.exceptions.HTTPError as e: + if e.response.status_code != 404: + raise e + # create prompt if it doesn't exist + # make sure I am owner if owner is specified + if ( + settings["tenant_handle"] + and owner != "-" + and settings["tenant_handle"] != owner + ): + raise ValueError( + f"Tenant {settings['tenant_handle']} is not the owner of repo {prompt_identifier}" + ) + self.create_prompt( + prompt_name, + is_public=is_public, + description=description, + ) + + manifest_dict = json.loads(manifest_json) + if parent_commit_hash == "latest": + parent_commit_hash = self._get_latest_commit_hash(prompt_full_name) + print('dict to submit', manifest_dict) + request_dict = {"parent_commit": parent_commit_hash, "manifest": manifest_dict} + res = requests.post( + f"{self.api_url}/commits/{prompt_full_name}", + headers=self._headers, + json=request_dict, + ) + res.raise_for_status() + res = res.json() + commit_hash = res["commit"]["commit_hash"] + short_hash = commit_hash[:8] + url = ( + self._host_url + + f"/prompts/{prompt_name}/{short_hash}?organizationId={settings['id']}" + ) + return url + + def _tracing_thread_drain_queue( tracing_queue: Queue, limit: int = 100, block: bool = True ) -> List[TracingQueueItem]: diff --git a/python/langsmith/schemas.py b/python/langsmith/schemas.py index 453aa13de..d8ea947bd 100644 --- a/python/langsmith/schemas.py +++ b/python/langsmith/schemas.py @@ -744,3 +744,41 @@ def metadata(self) -> dict[str, Any]: if self.extra is None or "metadata" not in self.extra: return {} return self.extra["metadata"] + + +class PromptManifest(BaseModel): + owner: str + repo: str + commit_hash: str + manifest: Dict[str, Any] + examples: List[dict] + + +class Prompt(BaseModel): + repo_handle: str + description: str | None + readme: str | None + id: str + tenant_id: str + created_at: datetime + updated_at: datetime + is_public: bool + is_archived: bool + tags: List[str] + original_repo_id: str | None + upstream_repo_id: str | None + owner: str + full_name: str + num_likes: int + num_downloads: int + num_views: int + liked_by_auth_user: bool + last_commit_hash: str | None + num_commits: int + original_repo_full_name: str | None + upstream_repo_full_name: str | None + + +class ListPromptsResponse(BaseModel): + repos: List[Prompt] + total: int diff --git a/python/langsmith/utils.py b/python/langsmith/utils.py index 2c0152e0f..b8ad4f6fc 100644 --- a/python/langsmith/utils.py +++ b/python/langsmith/utils.py @@ -561,3 +561,28 @@ def deepish_copy(val: T) -> T: # what we can _LOGGER.debug("Failed to deepcopy input: %s", repr(e)) return _middle_copy(val, memo) + + +def is_version_greater_or_equal(current_version, target_version): + from packaging import version + + current = version.parse(current_version) + target = version.parse(target_version) + return current >= target + + +def parse_prompt_identifier(identifier: str) -> Tuple[str, str, str]: + """ + Parses a string in the format of `owner/repo:commit` and returns a tuple of + (owner, repo, commit). + """ + owner_prompt = identifier + commit = "latest" + if ":" in identifier: + owner_prompt, commit = identifier.split(":", 1) + + if "/" not in owner_prompt: + return "-", owner_prompt, commit + + owner, prompt = owner_prompt.split("/", 1) + return owner, prompt, commit diff --git a/python/tests/hub/__init__.py b/python/tests/hub/__init__.py new file mode 100644 index 000000000..e69de29bb From 95eee541633698f94123bbad18cdfbfb7874a247 Mon Sep 17 00:00:00 2001 From: Maddy Adams Date: Wed, 10 Jul 2024 13:00:46 -0700 Subject: [PATCH 02/61] updates --- python/langsmith/client.py | 91 ++++++++++++++++++++++---------------- 1 file changed, 54 insertions(+), 37 deletions(-) diff --git a/python/langsmith/client.py b/python/langsmith/client.py index e67ec8e21..6d092f335 100644 --- a/python/langsmith/client.py +++ b/python/langsmith/client.py @@ -4590,9 +4590,21 @@ def get_prompt(self, prompt_identifier: str) -> ls_schemas.Prompt: return ls_schemas.Prompt(**prompt) + def current_tenant_is_owner(self, owner: str) -> bool: + settings = self.get_settings() + if owner != "-" and settings["tenant_handle"] != owner: + return False + return True + def create_prompt( - self, prompt_name: str, *, description: str = "", is_public: bool = True + self, owner: str, prompt_name: str, *, description: str = "", is_public: bool = True ): + if not self.current_tenant_is_owner(owner): + settings = self.get_settings() + raise ValueError( + f"Cannot create prompt for another tenant. Current tenant: {settings['tenant_handle'] or 'no handle'}, Requested tenant: {owner}" + ) + json = { "repo_handle": prompt_name, "is_public": is_public, @@ -4622,7 +4634,17 @@ def _get_latest_commit_hash(self, prompt_identifier: str) -> Optional[str]: if len(commits) == 0: return None return commits[0]["commit_hash"] + + def prompt_exists(self, prompt_name: str) -> bool: + try: + # check if the prompt exists + self.get_prompt(prompt_name) + return True + except requests.exceptions.HTTPError as e: + if e.response.status_code == 404: + return False + raise e def pull_prompt( self, @@ -4670,47 +4692,42 @@ def push_prompt( """Push a prompt to the LangSmith API. Args: - prompt_name: The name of the prompt + prompt_identifier: The name of the prompt in the format "prompt_name" or "owner/prompt_name" manifest_json: The JSON string of the prompt manifest - parent_commit_hash: The commit hash of the parent commit - is_public: Whether the new prompt is public - description: The description of the new prompt + parent_commit_hash: The commit hash of the parent commit, default is "latest" + is_public: Whether the new prompt is public, default is False + description: The description of the new prompt, default is an empty string """ from langchain_core.load.dump import dumps + manifest_json = dumps(manifest_json) settings = self.get_settings() - if is_public: - if not settings["tenant_handle"]: - raise ValueError( - """ - Cannot create public prompt without first creating a LangChain Hub handle. - - You can add a handle by creating a public prompt at: - https://smith.langchain.com/prompts - - This is a workspace-level handle and will be associated with all of your workspace's public prompts in the LangChain Hub. - """ - ) + + if is_public and not settings.get("tenant_handle"): + raise ValueError( + """ + Cannot create a public prompt without first creating a LangChain Hub handle. + + You can add a handle by creating a public prompt at: + https://smith.langchain.com/prompts + + This is a workspace-level handle and will be associated with all of your workspace's public prompts in the LangChain Hub. + """ + ) + owner, prompt_name, _ = ls_utils.parse_prompt_identifier(prompt_identifier) prompt_full_name = f"{owner}/{prompt_name}" - try: - # check if the prompt exists - _ = self.get_prompt(prompt_full_name) - except requests.exceptions.HTTPError as e: - if e.response.status_code != 404: - raise e - # create prompt if it doesn't exist - # make sure I am owner if owner is specified - if ( - settings["tenant_handle"] - and owner != "-" - and settings["tenant_handle"] != owner - ): - raise ValueError( - f"Tenant {settings['tenant_handle']} is not the owner of repo {prompt_identifier}" - ) + + if not self.current_tenant_is_owner(owner): + settings = self.get_settings() + raise ValueError( + f"Cannot create prompt for another tenant. Current tenant: {settings['tenant_handle'] or 'no handle'}, Requested tenant: {owner}" + ) + + if not self.prompt_exists(prompt_full_name): self.create_prompt( - prompt_name, + owner=owner, + prompt_name=prompt_name, is_public=is_public, description=description, ) @@ -4718,16 +4735,16 @@ def push_prompt( manifest_dict = json.loads(manifest_json) if parent_commit_hash == "latest": parent_commit_hash = self._get_latest_commit_hash(prompt_full_name) - print('dict to submit', manifest_dict) request_dict = {"parent_commit": parent_commit_hash, "manifest": manifest_dict} res = requests.post( f"{self.api_url}/commits/{prompt_full_name}", headers=self._headers, json=request_dict, ) + if res.status_code == 409: + raise ValueError("Conflict: The prompt has not been updated since the last commit") res.raise_for_status() - res = res.json() - commit_hash = res["commit"]["commit_hash"] + commit_hash = res.json()["commit"]["commit_hash"] short_hash = commit_hash[:8] url = ( self._host_url From 6030feed42f93e5a82446859f68a1ea58e94fce9 Mon Sep 17 00:00:00 2001 From: Maddy Adams Date: Thu, 11 Jul 2024 13:56:42 -0700 Subject: [PATCH 03/61] update --- python/Makefile | 3 + python/langsmith/client.py | 196 ++++++++-------------- python/tests/{hub => prompts}/__init__.py | 0 python/tests/prompts/test_prompts.py | 39 +++++ 4 files changed, 112 insertions(+), 126 deletions(-) rename python/tests/{hub => prompts}/__init__.py (100%) create mode 100644 python/tests/prompts/test_prompts.py diff --git a/python/Makefile b/python/Makefile index d06830bf9..a8bab2a27 100644 --- a/python/Makefile +++ b/python/Makefile @@ -18,6 +18,9 @@ doctest: evals: poetry run python -m pytest tests/evaluation +prompts: + poetry run python -m pytest tests/prompts + lint: poetry run ruff check . poetry run mypy . diff --git a/python/langsmith/client.py b/python/langsmith/client.py index 6d092f335..0bf8033e1 100644 --- a/python/langsmith/client.py +++ b/python/langsmith/client.py @@ -4562,83 +4562,38 @@ def _evaluate_strings( def get_settings(self): - res = requests.get( - f"{self.api_url}/settings", - headers=self._headers, - ) - res.raise_for_status() - return res.json() - - - def list_prompts(self, limit: int = 100, offset: int = 0) -> ls_schemas.ListPromptsResponse: - res = requests.get( - f"{self.api_url}/repos?limit={limit}&offset={offset}", - headers=self._headers, - ) - res.raise_for_status() - res_dict = res.json() - return ls_schemas.ListPromptsResponse(**res_dict) - - - def get_prompt(self, prompt_identifier: str) -> ls_schemas.Prompt: - res = requests.get( - f"{self.api_url}/repos/{prompt_identifier}", - headers=self._headers, - ) - res.raise_for_status() - prompt = res.json()['repo'] - return ls_schemas.Prompt(**prompt) + """ + Get the settings for the current tenant. + Returns: + dict: The settings for the current tenant. + """ + response = self.request_with_retries("GET", "/settings") + return response.json() + def current_tenant_is_owner(self, owner: str) -> bool: settings = self.get_settings() - if owner != "-" and settings["tenant_handle"] != owner: - return False - return True - - def create_prompt( - self, owner: str, prompt_name: str, *, description: str = "", is_public: bool = True - ): - if not self.current_tenant_is_owner(owner): - settings = self.get_settings() - raise ValueError( - f"Cannot create prompt for another tenant. Current tenant: {settings['tenant_handle'] or 'no handle'}, Requested tenant: {owner}" - ) + return owner == "-" or settings["tenant_handle"] == owner - json = { - "repo_handle": prompt_name, - "is_public": is_public, - "description": description, - } - res = requests.post( - f"{self.api_url}/repos/", - headers=self._headers, - json=json, - ) - res.raise_for_status() - return res.json() + def list_prompts(self, limit: int = 100, offset: int = 0) -> ls_schemas.ListPromptsResponse: + params = {"limit": limit, "offset": offset} + res_dict = self.request_with_retries("GET", "/repos", params=params) + return ls_schemas.ListPromptsResponse(**res_dict) - def list_commits(self, prompt_name: str, limit: int = 100, offset: int = 0): - res = requests.get( - f"{self.api_url}/commits/{prompt_name}/?limit={limit}&offset={offset}", - headers=self._headers, - ) - res.raise_for_status() - return res.json() + def get_prompt(self, prompt_identifier: str) -> ls_schemas.Prompt: + response = self.request_with_retries("GET", f"/repos/{prompt_identifier}") + if response.status_code == 200: + res = response.json() + return ls_schemas.Prompt(**res['repo']) + else: + response.raise_for_status() - def _get_latest_commit_hash(self, prompt_identifier: str) -> Optional[str]: - commits_resp = self.list_commits(prompt_identifier) - commits = commits_resp["commits"] - if len(commits) == 0: - return None - return commits[0]["commit_hash"] - def prompt_exists(self, prompt_name: str) -> bool: try: - # check if the prompt exists self.get_prompt(prompt_name) return True except requests.exceptions.HTTPError as e: @@ -4646,10 +4601,15 @@ def prompt_exists(self, prompt_name: str) -> bool: return False raise e - def pull_prompt( - self, - prompt_identifier: str, - ) -> ls_schemas.PromptManifest: + + def _get_latest_commit_hash(self, prompt_owner_and_name: str, limit: int = 1, offset: int = 0) -> Optional[str]: + response = self.request_with_retries("GET", f"/commits/{prompt_owner_and_name}/", params={"limit": limit, "offset": offset}) + commits_resp = response.json() + commits = commits_resp["commits"] + return commits[0]["commit_hash"] if commits else None + + + def pull_prompt_manifest(self, prompt_identifier: str) -> ls_schemas.PromptManifest: """Pull a prompt from the LangSmith API. Args: @@ -4659,45 +4619,37 @@ def pull_prompt( Prompt The prompt """ - LS_VERSION_WITH_OPTIMIZATION="0.5.23" + owner, prompt_name, commit_hash = ls_utils.parse_prompt_identifier(prompt_identifier) + use_optimization = ls_utils.is_version_greater_or_equal( - current_version=self.info.version, target_version=LS_VERSION_WITH_OPTIMIZATION + current_version=self.info.version, target_version="0.5.23" ) - owner, prompt_name, commit_hash = ls_utils.parse_prompt_identifier(prompt_identifier) + if not use_optimization and (commit_hash is None or commit_hash == "latest"): + commit_hash = self._get_latest_commit_hash(f"{owner}/{prompt_name}") + if commit_hash is None: + raise ValueError("No commits found") - if not use_optimization: - if commit_hash is None or commit_hash == "latest": - commit_hash = self._get_latest_commit_hash(f"{owner}/{prompt_name}") - if commit_hash is None: - raise ValueError("No commits found") + response = self.request_with_retries("GET", f"/commits/{owner}/{prompt_name}/{commit_hash}") + res = response.json() + return ls_schemas.PromptManifest(**{"owner": owner, "repo": prompt_name, **res}) - res = requests.get( - f"{self.api_url}/commits/{owner}/{prompt_name}/{commit_hash}", - headers=self._headers, - ) - res.raise_for_status() - result = res.json() - return ls_schemas.PromptManifest(**{"owner": owner, "repo": prompt_name, **result}) - - def push_prompt( - self, - prompt_identifier: str, - manifest_json: Any, - *, - parent_commit_hash: Optional[str] = "latest", - is_public: bool = False, - description: str = "", - ) -> str: - """Push a prompt to the LangSmith API. - Args: - prompt_identifier: The name of the prompt in the format "prompt_name" or "owner/prompt_name" - manifest_json: The JSON string of the prompt manifest - parent_commit_hash: The commit hash of the parent commit, default is "latest" - is_public: Whether the new prompt is public, default is False - description: The description of the new prompt, default is an empty string - """ + def pull_prompt(self, prompt_identifier: str) -> ls_schemas.PromptManifest: + from langchain_core.load.load import loads + from langchain_core.prompts import BasePromptTemplate + response = self.pull_prompt_manifest(prompt_identifier) + obj = loads(json.dumps(response.manifest)) + if isinstance(obj, BasePromptTemplate): + if obj.metadata is None: + obj.metadata = {} + obj.metadata["lc_hub_owner"] = response.owner + obj.metadata["lc_hub_repo"] = response.repo + obj.metadata["lc_hub_commit_hash"] = response.commit_hash + return obj + + + def push_prompt_manifest(self, prompt_identifier: str, manifest_json: Any, parent_commit_hash: Optional[str] = "latest", is_public: bool = False, description: str = ""): from langchain_core.load.dump import dumps manifest_json = dumps(manifest_json) @@ -4719,38 +4671,30 @@ def push_prompt( prompt_full_name = f"{owner}/{prompt_name}" if not self.current_tenant_is_owner(owner): - settings = self.get_settings() - raise ValueError( - f"Cannot create prompt for another tenant. Current tenant: {settings['tenant_handle'] or 'no handle'}, Requested tenant: {owner}" - ) + raise ValueError(f"Cannot create prompt for another tenant. Current tenant: {settings['tenant_handle'] or 'no handle'}, Requested tenant: {owner}") if not self.prompt_exists(prompt_full_name): - self.create_prompt( - owner=owner, - prompt_name=prompt_name, - is_public=is_public, - description=description, - ) + self.request_with_retries("POST", "/repos/", json = { + "repo_handle": prompt_name, + "is_public": is_public, + "description": description, + }) manifest_dict = json.loads(manifest_json) if parent_commit_hash == "latest": parent_commit_hash = self._get_latest_commit_hash(prompt_full_name) + request_dict = {"parent_commit": parent_commit_hash, "manifest": manifest_dict} - res = requests.post( - f"{self.api_url}/commits/{prompt_full_name}", - headers=self._headers, - json=request_dict, - ) - if res.status_code == 409: - raise ValueError("Conflict: The prompt has not been updated since the last commit") - res.raise_for_status() - commit_hash = res.json()["commit"]["commit_hash"] + response = self.request_with_retries("POST", f"/commits/{prompt_full_name}", json=request_dict) + res = response.json() + + commit_hash = res["commit"]["commit_hash"] short_hash = commit_hash[:8] - url = ( - self._host_url - + f"/prompts/{prompt_name}/{short_hash}?organizationId={settings['id']}" - ) - return url + return f"{self._host_url}/prompts/{prompt_name}/{short_hash}?organizationId={settings['id']}" + + + def push_prompt(self, prompt_identifier: str, obj: Any, parent_commit_hash: Optional[str] = "latest", is_public: bool = False, description: str = "") -> str: + return self.push_prompt_manifest(prompt_identifier, obj, parent_commit_hash, is_public, description) def _tracing_thread_drain_queue( diff --git a/python/tests/hub/__init__.py b/python/tests/prompts/__init__.py similarity index 100% rename from python/tests/hub/__init__.py rename to python/tests/prompts/__init__.py diff --git a/python/tests/prompts/test_prompts.py b/python/tests/prompts/test_prompts.py new file mode 100644 index 000000000..c018eadd0 --- /dev/null +++ b/python/tests/prompts/test_prompts.py @@ -0,0 +1,39 @@ +import asyncio +from typing import Sequence + +import pytest + +from langsmith import Client +from langsmith.schemas import Prompt + +from langchain_core.prompts import ChatPromptTemplate + +@pytest.fixture +def basic_fstring_prompt(): + return ChatPromptTemplate.from_messages( + [ + ("system", "You are a helpful asssistant."), + ("human", "{question}"), + ] + ) + +def test_push_prompt( + basic_fstring_prompt, +): + prompt_name = "basic_fstring_prompt" + langsmith_client = Client() + url = langsmith_client.push_prompt_manifest( + prompt_name, + basic_fstring_prompt + ) + assert prompt_name in url + + res = langsmith_client.push_prompt_manifest( + prompt_name, + basic_fstring_prompt + ) + assert res.status_code == 409 + + prompt = langsmith_client.pull_prompt_manifest(prompt_identifier=prompt_name) + assert prompt.repo == prompt_name + From f4b21a436cf87ef3619a889e1915f0e8bbb2d840 Mon Sep 17 00:00:00 2001 From: Maddy Adams Date: Thu, 11 Jul 2024 14:33:36 -0700 Subject: [PATCH 04/61] tests --- python/langsmith/client.py | 22 +++++-- python/tests/prompts/test_prompts.py | 95 ++++++++++++++++++++-------- 2 files changed, 86 insertions(+), 31 deletions(-) diff --git a/python/langsmith/client.py b/python/langsmith/client.py index 0bf8033e1..41660cd48 100644 --- a/python/langsmith/client.py +++ b/python/langsmith/client.py @@ -4579,12 +4579,14 @@ def current_tenant_is_owner(self, owner: str) -> bool: def list_prompts(self, limit: int = 100, offset: int = 0) -> ls_schemas.ListPromptsResponse: params = {"limit": limit, "offset": offset} - res_dict = self.request_with_retries("GET", "/repos", params=params) - return ls_schemas.ListPromptsResponse(**res_dict) + response = self.request_with_retries("GET", "/repos", params=params) + res = response.json() + return ls_schemas.ListPromptsResponse(**res) def get_prompt(self, prompt_identifier: str) -> ls_schemas.Prompt: - response = self.request_with_retries("GET", f"/repos/{prompt_identifier}") + owner, prompt_name, _ = ls_utils.parse_prompt_identifier(prompt_identifier) + response = self.request_with_retries("GET", f"/repos/{owner}/{prompt_name}", to_ignore=[ls_utils.LangSmithError]) if response.status_code == 200: res = response.json() return ls_schemas.Prompt(**res['repo']) @@ -4592,9 +4594,9 @@ def get_prompt(self, prompt_identifier: str) -> ls_schemas.Prompt: response.raise_for_status() - def prompt_exists(self, prompt_name: str) -> bool: + def prompt_exists(self, prompt_identifier: str) -> bool: try: - self.get_prompt(prompt_name) + self.get_prompt(prompt_identifier) return True except requests.exceptions.HTTPError as e: if e.response.status_code == 404: @@ -4607,6 +4609,14 @@ def _get_latest_commit_hash(self, prompt_owner_and_name: str, limit: int = 1, of commits_resp = response.json() commits = commits_resp["commits"] return commits[0]["commit_hash"] if commits else None + + + def delete_prompt(self, prompt_identifier: str) -> bool: + owner, prompt_name, _ = ls_utils.parse_prompt_identifier(prompt_identifier) + if not self.current_tenant_is_owner(owner): + raise ValueError(f"Cannot delete prompt for another tenant. Current tenant: {self.get_settings()['tenant_handle']}, Requested tenant: {owner}") + response = self.request_with_retries("DELETE", f"/repos/{owner}/{prompt_name}") + return response.status_code == 204 def pull_prompt_manifest(self, prompt_identifier: str) -> ls_schemas.PromptManifest: @@ -4635,7 +4645,7 @@ def pull_prompt_manifest(self, prompt_identifier: str) -> ls_schemas.PromptManif return ls_schemas.PromptManifest(**{"owner": owner, "repo": prompt_name, **res}) - def pull_prompt(self, prompt_identifier: str) -> ls_schemas.PromptManifest: + def pull_prompt(self, prompt_identifier: str) -> Any: from langchain_core.load.load import loads from langchain_core.prompts import BasePromptTemplate response = self.pull_prompt_manifest(prompt_identifier) diff --git a/python/tests/prompts/test_prompts.py b/python/tests/prompts/test_prompts.py index c018eadd0..2b8f69f0f 100644 --- a/python/tests/prompts/test_prompts.py +++ b/python/tests/prompts/test_prompts.py @@ -1,39 +1,84 @@ -import asyncio -from typing import Sequence - import pytest +from uuid import uuid4 +from langsmith.client import Client +from langsmith.schemas import Prompt, ListPromptsResponse +from langchain_core.prompts import ChatPromptTemplate -from langsmith import Client -from langsmith.schemas import Prompt +@pytest.fixture +def langsmith_client() -> Client: + return Client() -from langchain_core.prompts import ChatPromptTemplate +@pytest.fixture +def prompt_template_1() -> ChatPromptTemplate: + return ChatPromptTemplate.from_template("tell me a joke about {topic}") @pytest.fixture -def basic_fstring_prompt(): +def prompt_template_2() -> ChatPromptTemplate: return ChatPromptTemplate.from_messages( [ - ("system", "You are a helpful asssistant."), + ("system", "You are a helpful assistant."), ("human", "{question}"), ] ) -def test_push_prompt( - basic_fstring_prompt, -): - prompt_name = "basic_fstring_prompt" - langsmith_client = Client() - url = langsmith_client.push_prompt_manifest( - prompt_name, - basic_fstring_prompt - ) - assert prompt_name in url +def test_list_prompts(langsmith_client: Client): + # Test listing prompts + response = langsmith_client.list_prompts(limit=10, offset=0) + assert isinstance(response, ListPromptsResponse) + assert len(response.repos) <= 10 - res = langsmith_client.push_prompt_manifest( - prompt_name, - basic_fstring_prompt - ) - assert res.status_code == 409 +def test_get_prompt(langsmith_client: Client, prompt_template_1: ChatPromptTemplate): + # First, create a prompt to test with + prompt_name = f"test_prompt_{uuid4().hex[:8]}" + langsmith_client.push_prompt(prompt_name, prompt_template_1) + + # Now test getting the prompt + prompt = langsmith_client.get_prompt(prompt_name) + assert isinstance(prompt, Prompt) + assert prompt.repo_handle == prompt_name + + # Clean up + langsmith_client.delete_prompt(prompt_name) + assert not langsmith_client.prompt_exists(prompt_name) + +def test_prompt_exists(langsmith_client: Client, prompt_template_2: ChatPromptTemplate): + # Test with a non-existent prompt + non_existent_prompt = f"non_existent_{uuid4().hex[:8]}" + assert not langsmith_client.prompt_exists(non_existent_prompt) + + # Create a prompt and test again + existent_prompt = f"existent_{uuid4().hex[:8]}" + langsmith_client.push_prompt(existent_prompt, prompt_template_2) + assert langsmith_client.prompt_exists(existent_prompt) + + # Clean up + langsmith_client.delete_prompt(existent_prompt) + assert not langsmith_client.prompt_exists(existent_prompt) + +def test_push_and_pull_prompt(langsmith_client: Client, prompt_template_2: ChatPromptTemplate): + prompt_name = f"test_prompt_{uuid4().hex[:8]}" + + # Test pushing a prompt + push_result = langsmith_client.push_prompt(prompt_name, prompt_template_2) + assert isinstance(push_result, str) # Should return a URL + + # Test pulling the prompt + langsmith_client.pull_prompt(prompt_name) + + # Clean up + langsmith_client.delete_prompt(prompt_name) + +def test_push_prompt_manifest(langsmith_client: Client, prompt_template_2: ChatPromptTemplate): + prompt_name = f"test_prompt_manifest_{uuid4().hex[:8]}" + + # Test pushing a prompt manifest + result = langsmith_client.push_prompt_manifest(prompt_name, prompt_template_2) + assert isinstance(result, str) # Should return a URL - prompt = langsmith_client.pull_prompt_manifest(prompt_identifier=prompt_name) - assert prompt.repo == prompt_name + # Verify the pushed manifest + pulled_prompt_manifest = langsmith_client.pull_prompt_manifest(prompt_name) + latest_commit_hash = langsmith_client._get_latest_commit_hash(f"-/{prompt_name}") + assert pulled_prompt_manifest.commit_hash == latest_commit_hash + # Clean up + langsmith_client.delete_prompt(prompt_name) From b65bcf19b0ccef0c61907b1d6b0e55db2142fb90 Mon Sep 17 00:00:00 2001 From: Maddy Adams Date: Thu, 11 Jul 2024 14:53:21 -0700 Subject: [PATCH 05/61] docstrings --- python/langsmith/client.py | 167 +++++++++++++++++++++++++++++-------- 1 file changed, 130 insertions(+), 37 deletions(-) diff --git a/python/langsmith/client.py b/python/langsmith/client.py index 41660cd48..16123b634 100644 --- a/python/langsmith/client.py +++ b/python/langsmith/client.py @@ -4561,7 +4561,8 @@ def _evaluate_strings( ) - def get_settings(self): +class Client: + def get_settings(self) -> dict: """ Get the settings for the current tenant. @@ -4571,47 +4572,103 @@ def get_settings(self): response = self.request_with_retries("GET", "/settings") return response.json() - + def current_tenant_is_owner(self, owner: str) -> bool: + """ + Check if the current tenant is the owner of the prompt. + + Args: + owner (str): The owner to check against. + + Returns: + bool: True if the current tenant is the owner, False otherwise. + """ settings = self.get_settings() return owner == "-" or settings["tenant_handle"] == owner def list_prompts(self, limit: int = 100, offset: int = 0) -> ls_schemas.ListPromptsResponse: + """ + List prompts with pagination. + + Args: + limit (int): The maximum number of prompts to return. Defaults to 100. + offset (int): The number of prompts to skip. Defaults to 0. + + Returns: + ls_schemas.ListPromptsResponse: A response object containing the list of prompts. + """ params = {"limit": limit, "offset": offset} response = self.request_with_retries("GET", "/repos", params=params) - res = response.json() - return ls_schemas.ListPromptsResponse(**res) - + return ls_schemas.ListPromptsResponse(**response.json()) def get_prompt(self, prompt_identifier: str) -> ls_schemas.Prompt: + """ + Get a specific prompt by its identifier. + + Args: + prompt_identifier (str): The identifier of the prompt. + + Returns: + ls_schemas.Prompt: The prompt object. + + Raises: + requests.exceptions.HTTPError: If the prompt is not found or another error occurs. + """ owner, prompt_name, _ = ls_utils.parse_prompt_identifier(prompt_identifier) response = self.request_with_retries("GET", f"/repos/{owner}/{prompt_name}", to_ignore=[ls_utils.LangSmithError]) if response.status_code == 200: - res = response.json() - return ls_schemas.Prompt(**res['repo']) - else: - response.raise_for_status() + return ls_schemas.Prompt(**response.json()['repo']) + response.raise_for_status() def prompt_exists(self, prompt_identifier: str) -> bool: + """ + Check if a prompt exists. + + Args: + prompt_identifier (str): The identifier of the prompt. + + Returns: + bool: True if the prompt exists, False otherwise. + """ try: self.get_prompt(prompt_identifier) return True except requests.exceptions.HTTPError as e: - if e.response.status_code == 404: - return False - raise e + return e.response.status_code != 404 def _get_latest_commit_hash(self, prompt_owner_and_name: str, limit: int = 1, offset: int = 0) -> Optional[str]: + """ + Get the latest commit hash for a prompt. + + Args: + prompt_owner_and_name (str): The owner and name of the prompt. + limit (int): The maximum number of commits to fetch. Defaults to 1. + offset (int): The number of commits to skip. Defaults to 0. + + Returns: + Optional[str]: The latest commit hash, or None if no commits are found. + """ response = self.request_with_retries("GET", f"/commits/{prompt_owner_and_name}/", params={"limit": limit, "offset": offset}) - commits_resp = response.json() - commits = commits_resp["commits"] + commits = response.json()["commits"] return commits[0]["commit_hash"] if commits else None - + def delete_prompt(self, prompt_identifier: str) -> bool: + """ + Delete a prompt. + + Args: + prompt_identifier (str): The identifier of the prompt to delete. + + Returns: + bool: True if the prompt was successfully deleted, False otherwise. + + Raises: + ValueError: If the current tenant is not the owner of the prompt. + """ owner, prompt_name, _ = ls_utils.parse_prompt_identifier(prompt_identifier) if not self.current_tenant_is_owner(owner): raise ValueError(f"Cannot delete prompt for another tenant. Current tenant: {self.get_settings()['tenant_handle']}, Requested tenant: {owner}") @@ -4620,20 +4677,20 @@ def delete_prompt(self, prompt_identifier: str) -> bool: def pull_prompt_manifest(self, prompt_identifier: str) -> ls_schemas.PromptManifest: - """Pull a prompt from the LangSmith API. + """ + Pull a prompt manifest from the LangSmith API. Args: - prompt_identifier: The identifier of the prompt (str, ex. "prompt_name", "owner/prompt_name", "owner/prompt_name:commit_hash") + prompt_identifier (str): The identifier of the prompt. - Yields: - Prompt - The prompt + Returns: + ls_schemas.PromptManifest: The prompt manifest. + + Raises: + ValueError: If no commits are found for the prompt. """ owner, prompt_name, commit_hash = ls_utils.parse_prompt_identifier(prompt_identifier) - - use_optimization = ls_utils.is_version_greater_or_equal( - current_version=self.info.version, target_version="0.5.23" - ) + use_optimization = ls_utils.is_version_greater_or_equal(self.info.version, "0.5.23") if not use_optimization and (commit_hash is None or commit_hash == "latest"): commit_hash = self._get_latest_commit_hash(f"{owner}/{prompt_name}") @@ -4646,6 +4703,15 @@ def pull_prompt_manifest(self, prompt_identifier: str) -> ls_schemas.PromptManif def pull_prompt(self, prompt_identifier: str) -> Any: + """ + Pull a prompt and return it as a LangChain object. + + Args: + prompt_identifier (str): The identifier of the prompt. + + Returns: + Any: The prompt object. + """ from langchain_core.load.load import loads from langchain_core.prompts import BasePromptTemplate response = self.pull_prompt_manifest(prompt_identifier) @@ -4653,13 +4719,31 @@ def pull_prompt(self, prompt_identifier: str) -> Any: if isinstance(obj, BasePromptTemplate): if obj.metadata is None: obj.metadata = {} - obj.metadata["lc_hub_owner"] = response.owner - obj.metadata["lc_hub_repo"] = response.repo - obj.metadata["lc_hub_commit_hash"] = response.commit_hash + obj.metadata.update({ + "lc_hub_owner": response.owner, + "lc_hub_repo": response.repo, + "lc_hub_commit_hash": response.commit_hash + }) return obj - def push_prompt_manifest(self, prompt_identifier: str, manifest_json: Any, parent_commit_hash: Optional[str] = "latest", is_public: bool = False, description: str = ""): + def push_prompt_manifest(self, prompt_identifier: str, manifest_json: Any, parent_commit_hash: Optional[str] = "latest", is_public: bool = False, description: str = "") -> str: + """ + Push a prompt manifest to the LangSmith API. + + Args: + prompt_identifier (str): The identifier of the prompt. + manifest_json (Any): The manifest to push. + parent_commit_hash (Optional[str]): The parent commit hash. Defaults to "latest". + is_public (bool): Whether the prompt should be public. Defaults to False. + description (str): A description of the prompt. Defaults to an empty string. + + Returns: + str: The URL of the pushed prompt. + + Raises: + ValueError: If a public prompt is attempted without a tenant handle or if the current tenant is not the owner. + """ from langchain_core.load.dump import dumps manifest_json = dumps(manifest_json) @@ -4667,14 +4751,8 @@ def push_prompt_manifest(self, prompt_identifier: str, manifest_json: Any, paren if is_public and not settings.get("tenant_handle"): raise ValueError( - """ - Cannot create a public prompt without first creating a LangChain Hub handle. - - You can add a handle by creating a public prompt at: - https://smith.langchain.com/prompts - - This is a workspace-level handle and will be associated with all of your workspace's public prompts in the LangChain Hub. - """ + "Cannot create a public prompt without first creating a LangChain Hub handle. " + "You can add a handle by creating a public prompt at: https://smith.langchain.com/prompts" ) owner, prompt_name, _ = ls_utils.parse_prompt_identifier(prompt_identifier) @@ -4684,7 +4762,7 @@ def push_prompt_manifest(self, prompt_identifier: str, manifest_json: Any, paren raise ValueError(f"Cannot create prompt for another tenant. Current tenant: {settings['tenant_handle'] or 'no handle'}, Requested tenant: {owner}") if not self.prompt_exists(prompt_full_name): - self.request_with_retries("POST", "/repos/", json = { + self.request_with_retries("POST", "/repos/", json={ "repo_handle": prompt_name, "is_public": is_public, "description": description, @@ -4704,6 +4782,21 @@ def push_prompt_manifest(self, prompt_identifier: str, manifest_json: Any, paren def push_prompt(self, prompt_identifier: str, obj: Any, parent_commit_hash: Optional[str] = "latest", is_public: bool = False, description: str = "") -> str: + """ + Push a prompt object to the LangSmith API. + + This method is a wrapper around push_prompt_manifest. + + Args: + prompt_identifier (str): The identifier of the prompt. The format is "name" or "-/name" or "workspace_handle/name". + obj (Any): The prompt object to push. + parent_commit_hash (Optional[str]): The parent commit hash. Defaults to "latest". + is_public (bool): Whether the prompt should be public. Defaults to False. + description (str): A description of the prompt. Defaults to an empty string. + + Returns: + str: The URL of the pushed prompt. + """ return self.push_prompt_manifest(prompt_identifier, obj, parent_commit_hash, is_public, description) From 9ed7046b389e816835616ecfd1fd23157deff61f Mon Sep 17 00:00:00 2001 From: Maddy Adams Date: Thu, 11 Jul 2024 14:54:07 -0700 Subject: [PATCH 06/61] mistake --- python/langsmith/client.py | 1 - 1 file changed, 1 deletion(-) diff --git a/python/langsmith/client.py b/python/langsmith/client.py index 16123b634..1834ef5df 100644 --- a/python/langsmith/client.py +++ b/python/langsmith/client.py @@ -4561,7 +4561,6 @@ def _evaluate_strings( ) -class Client: def get_settings(self) -> dict: """ Get the settings for the current tenant. From 9ee2f6eddace98f48ce68439e8158e934b89eaa0 Mon Sep 17 00:00:00 2001 From: Maddy Adams Date: Thu, 11 Jul 2024 14:54:56 -0700 Subject: [PATCH 07/61] unnecessary file --- python/tests/prompts/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 python/tests/prompts/__init__.py diff --git a/python/tests/prompts/__init__.py b/python/tests/prompts/__init__.py deleted file mode 100644 index e69de29bb..000000000 From b739f4c83ab4774694e21ff3645220540c5523c4 Mon Sep 17 00:00:00 2001 From: Maddy Adams Date: Thu, 11 Jul 2024 15:24:50 -0700 Subject: [PATCH 08/61] whitespace --- python/tests/prompts/test_prompts.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/python/tests/prompts/test_prompts.py b/python/tests/prompts/test_prompts.py index 2b8f69f0f..4554314f4 100644 --- a/python/tests/prompts/test_prompts.py +++ b/python/tests/prompts/test_prompts.py @@ -4,14 +4,17 @@ from langsmith.schemas import Prompt, ListPromptsResponse from langchain_core.prompts import ChatPromptTemplate + @pytest.fixture def langsmith_client() -> Client: return Client() + @pytest.fixture def prompt_template_1() -> ChatPromptTemplate: return ChatPromptTemplate.from_template("tell me a joke about {topic}") + @pytest.fixture def prompt_template_2() -> ChatPromptTemplate: return ChatPromptTemplate.from_messages( @@ -21,12 +24,14 @@ def prompt_template_2() -> ChatPromptTemplate: ] ) + def test_list_prompts(langsmith_client: Client): # Test listing prompts response = langsmith_client.list_prompts(limit=10, offset=0) assert isinstance(response, ListPromptsResponse) assert len(response.repos) <= 10 + def test_get_prompt(langsmith_client: Client, prompt_template_1: ChatPromptTemplate): # First, create a prompt to test with prompt_name = f"test_prompt_{uuid4().hex[:8]}" @@ -41,6 +46,7 @@ def test_get_prompt(langsmith_client: Client, prompt_template_1: ChatPromptTempl langsmith_client.delete_prompt(prompt_name) assert not langsmith_client.prompt_exists(prompt_name) + def test_prompt_exists(langsmith_client: Client, prompt_template_2: ChatPromptTemplate): # Test with a non-existent prompt non_existent_prompt = f"non_existent_{uuid4().hex[:8]}" @@ -55,6 +61,7 @@ def test_prompt_exists(langsmith_client: Client, prompt_template_2: ChatPromptTe langsmith_client.delete_prompt(existent_prompt) assert not langsmith_client.prompt_exists(existent_prompt) + def test_push_and_pull_prompt(langsmith_client: Client, prompt_template_2: ChatPromptTemplate): prompt_name = f"test_prompt_{uuid4().hex[:8]}" @@ -68,6 +75,7 @@ def test_push_and_pull_prompt(langsmith_client: Client, prompt_template_2: ChatP # Clean up langsmith_client.delete_prompt(prompt_name) + def test_push_prompt_manifest(langsmith_client: Client, prompt_template_2: ChatPromptTemplate): prompt_name = f"test_prompt_manifest_{uuid4().hex[:8]}" From bf521dc4b0c54d5dc8800e2f3b9fed042b70e1aa Mon Sep 17 00:00:00 2001 From: Maddy Adams Date: Thu, 11 Jul 2024 19:24:15 -0700 Subject: [PATCH 09/61] more functionality --- python/langsmith/client.py | 260 ++++++++++++++++++-------- python/langsmith/schemas.py | 46 ++++- python/langsmith/utils.py | 43 +++-- python/langsmith/wrappers/_openai.py | 12 +- python/tests/prompts/test_prompts.py | 131 ++++++++++--- python/tests/unit_tests/test_utils.py | 61 ++++-- 6 files changed, 420 insertions(+), 133 deletions(-) diff --git a/python/langsmith/client.py b/python/langsmith/client.py index 1834ef5df..86a13b107 100644 --- a/python/langsmith/client.py +++ b/python/langsmith/client.py @@ -4560,10 +4560,8 @@ def _evaluate_strings( **kwargs, ) - def get_settings(self) -> dict: - """ - Get the settings for the current tenant. + """Get the settings for the current tenant. Returns: dict: The settings for the current tenant. @@ -4571,10 +4569,8 @@ def get_settings(self) -> dict: response = self.request_with_retries("GET", "/settings") return response.json() - def current_tenant_is_owner(self, owner: str) -> bool: - """ - Check if the current tenant is the owner of the prompt. + """Check if the current workspace has the same handle as owner. Args: owner (str): The owner to check against. @@ -4585,79 +4581,164 @@ def current_tenant_is_owner(self, owner: str) -> bool: settings = self.get_settings() return owner == "-" or settings["tenant_handle"] == owner + def prompt_exists(self, prompt_identifier: str) -> bool: + """Check if a prompt exists. - def list_prompts(self, limit: int = 100, offset: int = 0) -> ls_schemas.ListPromptsResponse: + Args: + prompt_identifier (str): The identifier of the prompt. + + Returns: + bool: True if the prompt exists, False otherwise. """ - List prompts with pagination. + try: + self.get_prompt(prompt_identifier) + return True + except requests.exceptions.HTTPError as e: + return e.response.status_code != 404 + + def _get_latest_commit_hash( + self, prompt_owner_and_name: str, limit: int = 1, offset: int = 0 + ) -> Optional[str]: + """Get the latest commit hash for a prompt. Args: - limit (int): The maximum number of prompts to return. Defaults to 100. - offset (int): The number of prompts to skip. Defaults to 0. + prompt_owner_and_name (str): The owner and name of the prompt. + limit (int): The maximum number of commits to fetch. Defaults to 1. + offset (int): The number of commits to skip. Defaults to 0. Returns: - ls_schemas.ListPromptsResponse: A response object containing the list of prompts. + Optional[str]: The latest commit hash, or None if no commits are found. """ - params = {"limit": limit, "offset": offset} - response = self.request_with_retries("GET", "/repos", params=params) - return ls_schemas.ListPromptsResponse(**response.json()) + response = self.request_with_retries( + "GET", + f"/commits/{prompt_owner_and_name}/", + params={"limit": limit, "offset": offset}, + ) + commits = response.json()["commits"] + return commits[0]["commit_hash"] if commits else None - def get_prompt(self, prompt_identifier: str) -> ls_schemas.Prompt: - """ - Get a specific prompt by its identifier. + def _like_or_unlike_prompt( + self, prompt_identifier: str, like: bool + ) -> Dict[str, int]: + """Like or unlike a prompt. Args: prompt_identifier (str): The identifier of the prompt. + like (bool): True to like the prompt, False to unlike it. Returns: - ls_schemas.Prompt: The prompt object. + A dictionary with the key 'likes' and the count of likes as the value. Raises: requests.exceptions.HTTPError: If the prompt is not found or another error occurs. """ owner, prompt_name, _ = ls_utils.parse_prompt_identifier(prompt_identifier) - response = self.request_with_retries("GET", f"/repos/{owner}/{prompt_name}", to_ignore=[ls_utils.LangSmithError]) - if response.status_code == 200: - return ls_schemas.Prompt(**response.json()['repo']) + response = self.request_with_retries( + "POST", f"/likes/{owner}/{prompt_name}", json={"like": like} + ) response.raise_for_status() + return response.json + def like_prompt(self, prompt_identifier: str) -> Dict[str, int]: + """Check if a prompt exists. + + Args: + prompt_identifier (str): The identifier of the prompt. + + Returns: + A dictionary with the key 'likes' and the count of likes as the value. - def prompt_exists(self, prompt_identifier: str) -> bool: """ - Check if a prompt exists. + return self._like_or_unlike_prompt(prompt_identifier, like=True) + + def unlike_prompt(self, prompt_identifier: str) -> Dict[str, int]: + """Unlike a prompt. Args: prompt_identifier (str): The identifier of the prompt. Returns: - bool: True if the prompt exists, False otherwise. + A dictionary with the key 'likes' and the count of likes as the value. + """ - try: - self.get_prompt(prompt_identifier) - return True - except requests.exceptions.HTTPError as e: - return e.response.status_code != 404 + return self._like_or_unlike_prompt(prompt_identifier, like=False) + def list_prompts( + self, limit: int = 100, offset: int = 0 + ) -> ls_schemas.ListPromptsResponse: + """List prompts with pagination. + + Args: + limit (int): The maximum number of prompts to return. Defaults to 100. + offset (int): The number of prompts to skip. Defaults to 0. - def _get_latest_commit_hash(self, prompt_owner_and_name: str, limit: int = 1, offset: int = 0) -> Optional[str]: + Returns: + ls_schemas.ListPromptsResponse: A response object containing the list of prompts. """ - Get the latest commit hash for a prompt. + params = {"limit": limit, "offset": offset} + response = self.request_with_retries("GET", "/repos", params=params) + return ls_schemas.ListPromptsResponse(**response.json()) + + def get_prompt(self, prompt_identifier: str) -> ls_schemas.Prompt: + """Get a specific prompt by its identifier. Args: - prompt_owner_and_name (str): The owner and name of the prompt. - limit (int): The maximum number of commits to fetch. Defaults to 1. - offset (int): The number of commits to skip. Defaults to 0. + prompt_identifier (str): The identifier of the prompt. The identifier should be in the format "prompt_name" or "owner/prompt_name". Returns: - Optional[str]: The latest commit hash, or None if no commits are found. + ls_schemas.Prompt: The prompt object. + + Raises: + requests.exceptions.HTTPError: If the prompt is not found or another error occurs. """ - response = self.request_with_retries("GET", f"/commits/{prompt_owner_and_name}/", params={"limit": limit, "offset": offset}) - commits = response.json()["commits"] - return commits[0]["commit_hash"] if commits else None + owner, prompt_name, _ = ls_utils.parse_prompt_identifier(prompt_identifier) + response = self.request_with_retries( + "GET", f"/repos/{owner}/{prompt_name}", to_ignore=[ls_utils.LangSmithError] + ) + if response.status_code == 200: + return ls_schemas.Prompt(**response.json()["repo"]) + response.raise_for_status() + def update_prompt( + self, + prompt_identifier: str, + *, + description: Optional[str] = None, + is_public: Optional[bool] = None, + tags: Optional[List[str]] = None, + ) -> Dict[str, Any]: + """Update a prompt's metadata. - def delete_prompt(self, prompt_identifier: str) -> bool: + Args: + prompt_identifier (str): The identifier of the prompt to update. + description (Optional[str]): New description for the prompt. + is_public (Optional[bool]): New public status for the prompt. + tags (Optional[List[str]]): New list of tags for the prompt. + + Returns: + Dict[str, Any]: The updated prompt data as returned by the server. + + Raises: + ValueError: If the prompt_identifier is empty. + HTTPError: If the server request fails. """ - Delete a prompt. + json: Dict[str, Union[str, bool, List[str]]] = {} + if description is not None: + json["description"] = description + if is_public is not None: + json["is_public"] = is_public + if tags is not None: + json["tags"] = tags + + owner, prompt_name, _ = ls_utils.parse_prompt_identifier(prompt_identifier) + response = self.request_with_retries( + "PATCH", f"/repos/{owner}/{prompt_name}", json=json + ) + response.raise_for_status() + return response.json() + + def delete_prompt(self, prompt_identifier: str) -> bool: + """Delete a prompt. Args: prompt_identifier (str): The identifier of the prompt to delete. @@ -4670,14 +4751,14 @@ def delete_prompt(self, prompt_identifier: str) -> bool: """ owner, prompt_name, _ = ls_utils.parse_prompt_identifier(prompt_identifier) if not self.current_tenant_is_owner(owner): - raise ValueError(f"Cannot delete prompt for another tenant. Current tenant: {self.get_settings()['tenant_handle']}, Requested tenant: {owner}") + raise ValueError( + f"Cannot delete prompt for another tenant. Current tenant: {self.get_settings()['tenant_handle']}, Requested tenant: {owner}" + ) response = self.request_with_retries("DELETE", f"/repos/{owner}/{prompt_name}") return response.status_code == 204 - def pull_prompt_manifest(self, prompt_identifier: str) -> ls_schemas.PromptManifest: - """ - Pull a prompt manifest from the LangSmith API. + """Pull a prompt manifest from the LangSmith API. Args: prompt_identifier (str): The identifier of the prompt. @@ -4688,22 +4769,27 @@ def pull_prompt_manifest(self, prompt_identifier: str) -> ls_schemas.PromptManif Raises: ValueError: If no commits are found for the prompt. """ - owner, prompt_name, commit_hash = ls_utils.parse_prompt_identifier(prompt_identifier) - use_optimization = ls_utils.is_version_greater_or_equal(self.info.version, "0.5.23") + owner, prompt_name, commit_hash = ls_utils.parse_prompt_identifier( + prompt_identifier + ) + use_optimization = ls_utils.is_version_greater_or_equal( + self.info.version, "0.5.23" + ) if not use_optimization and (commit_hash is None or commit_hash == "latest"): commit_hash = self._get_latest_commit_hash(f"{owner}/{prompt_name}") if commit_hash is None: raise ValueError("No commits found") - response = self.request_with_retries("GET", f"/commits/{owner}/{prompt_name}/{commit_hash}") - res = response.json() - return ls_schemas.PromptManifest(**{"owner": owner, "repo": prompt_name, **res}) - + response = self.request_with_retries( + "GET", f"/commits/{owner}/{prompt_name}/{commit_hash}" + ) + return ls_schemas.PromptManifest( + **{"owner": owner, "repo": prompt_name, **response.json()} + ) def pull_prompt(self, prompt_identifier: str) -> Any: - """ - Pull a prompt and return it as a LangChain object. + """Pull a prompt and return it as a LangChain object. Args: prompt_identifier (str): The identifier of the prompt. @@ -4713,22 +4799,30 @@ def pull_prompt(self, prompt_identifier: str) -> Any: """ from langchain_core.load.load import loads from langchain_core.prompts import BasePromptTemplate + response = self.pull_prompt_manifest(prompt_identifier) obj = loads(json.dumps(response.manifest)) if isinstance(obj, BasePromptTemplate): if obj.metadata is None: obj.metadata = {} - obj.metadata.update({ - "lc_hub_owner": response.owner, - "lc_hub_repo": response.repo, - "lc_hub_commit_hash": response.commit_hash - }) + obj.metadata.update( + { + "lc_hub_owner": response.owner, + "lc_hub_repo": response.repo, + "lc_hub_commit_hash": response.commit_hash, + } + ) return obj - - def push_prompt_manifest(self, prompt_identifier: str, manifest_json: Any, parent_commit_hash: Optional[str] = "latest", is_public: bool = False, description: str = "") -> str: - """ - Push a prompt manifest to the LangSmith API. + def push_prompt_manifest( + self, + prompt_identifier: str, + manifest_json: Any, + parent_commit_hash: Optional[str] = "latest", + is_public: bool = False, + description: str = "", + ) -> str: + """Push a prompt manifest to the LangSmith API. Args: prompt_identifier (str): The identifier of the prompt. @@ -4758,31 +4852,43 @@ def push_prompt_manifest(self, prompt_identifier: str, manifest_json: Any, paren prompt_full_name = f"{owner}/{prompt_name}" if not self.current_tenant_is_owner(owner): - raise ValueError(f"Cannot create prompt for another tenant. Current tenant: {settings['tenant_handle'] or 'no handle'}, Requested tenant: {owner}") + raise ValueError( + f"Cannot create prompt for another tenant. Current tenant: {settings['tenant_handle'] or 'no handle'}, Requested tenant: {owner}" + ) if not self.prompt_exists(prompt_full_name): - self.request_with_retries("POST", "/repos/", json={ - "repo_handle": prompt_name, - "is_public": is_public, - "description": description, - }) + self.request_with_retries( + "POST", + "/repos/", + json={ + "repo_handle": prompt_name, + "is_public": is_public, + "description": description, + }, + ) manifest_dict = json.loads(manifest_json) if parent_commit_hash == "latest": parent_commit_hash = self._get_latest_commit_hash(prompt_full_name) - + request_dict = {"parent_commit": parent_commit_hash, "manifest": manifest_dict} - response = self.request_with_retries("POST", f"/commits/{prompt_full_name}", json=request_dict) - res = response.json() + response = self.request_with_retries( + "POST", f"/commits/{prompt_full_name}", json=request_dict + ) - commit_hash = res["commit"]["commit_hash"] + commit_hash = response.json()["commit"]["commit_hash"] short_hash = commit_hash[:8] return f"{self._host_url}/prompts/{prompt_name}/{short_hash}?organizationId={settings['id']}" - - def push_prompt(self, prompt_identifier: str, obj: Any, parent_commit_hash: Optional[str] = "latest", is_public: bool = False, description: str = "") -> str: - """ - Push a prompt object to the LangSmith API. + def push_prompt( + self, + prompt_identifier: str, + obj: Any, + parent_commit_hash: Optional[str] = "latest", + is_public: bool = False, + description: str = "", + ) -> str: + """Push a prompt object to the LangSmith API. This method is a wrapper around push_prompt_manifest. @@ -4796,7 +4902,9 @@ def push_prompt(self, prompt_identifier: str, obj: Any, parent_commit_hash: Opti Returns: str: The URL of the pushed prompt. """ - return self.push_prompt_manifest(prompt_identifier, obj, parent_commit_hash, is_public, description) + return self.push_prompt_manifest( + prompt_identifier, obj, parent_commit_hash, is_public, description + ) def _tracing_thread_drain_queue( diff --git a/python/langsmith/schemas.py b/python/langsmith/schemas.py index d8ea947bd..ebebfcb63 100644 --- a/python/langsmith/schemas.py +++ b/python/langsmith/schemas.py @@ -747,38 +747,80 @@ def metadata(self) -> dict[str, Any]: class PromptManifest(BaseModel): + """Represents a Prompt with a manifest. + + Attributes: + repo (str): The name of the prompt. + commit_hash (str): The commit hash of the prompt. + manifest (Dict[str, Any]): The manifest of the prompt. + examples (List[dict]): The list of examples. + """ + owner: str + """The handle of the owner of the prompt.""" repo: str + """The name of the prompt.""" commit_hash: str + """The commit hash of the prompt.""" manifest: Dict[str, Any] + """The manifest of the prompt.""" examples: List[dict] + """The list of examples.""" class Prompt(BaseModel): + """Represents a Prompt with metadata.""" + + owner: str + """The handle of the owner of the prompt.""" repo_handle: str + """The name of the prompt.""" + full_name: str + """The full name of the prompt. (owner + repo_handle)""" description: str | None + """The description of the prompt.""" readme: str | None + """The README of the prompt.""" id: str + """The ID of the prompt.""" tenant_id: str + """The tenant ID of the prompt owner.""" created_at: datetime + """The creation time of the prompt.""" updated_at: datetime + """The last update time of the prompt.""" is_public: bool + """Whether the prompt is public.""" is_archived: bool + """Whether the prompt is archived.""" tags: List[str] + """The tags associated with the prompt.""" original_repo_id: str | None + """The ID of the original prompt, if forked.""" upstream_repo_id: str | None - owner: str - full_name: str + """The ID of the upstream prompt, if forked.""" num_likes: int + """The number of likes.""" num_downloads: int + """The number of downloads.""" num_views: int + """The number of views.""" liked_by_auth_user: bool + """Whether the prompt is liked by the authenticated user.""" last_commit_hash: str | None + """The hash of the last commit.""" num_commits: int + """The number of commits.""" original_repo_full_name: str | None + """The full name of the original prompt, if forked.""" upstream_repo_full_name: str | None + """The full name of the upstream prompt, if forked.""" class ListPromptsResponse(BaseModel): + """A list of prompts with metadata.""" + repos: List[Prompt] + """The list of prompts.""" total: int + """The total number of prompts.""" diff --git a/python/langsmith/utils.py b/python/langsmith/utils.py index b8ad4f6fc..6fb0f0ff9 100644 --- a/python/langsmith/utils.py +++ b/python/langsmith/utils.py @@ -564,6 +564,7 @@ def deepish_copy(val: T) -> T: def is_version_greater_or_equal(current_version, target_version): + """Check if the current version is greater or equal to the target version.""" from packaging import version current = version.parse(current_version) @@ -572,17 +573,35 @@ def is_version_greater_or_equal(current_version, target_version): def parse_prompt_identifier(identifier: str) -> Tuple[str, str, str]: - """ - Parses a string in the format of `owner/repo:commit` and returns a tuple of - (owner, repo, commit). - """ - owner_prompt = identifier - commit = "latest" - if ":" in identifier: - owner_prompt, commit = identifier.split(":", 1) + """Parse a string in the format of `owner/name[:commit]` or `name[:commit]` and returns a tuple of (owner, name, commit). + + Args: + identifier (str): The prompt identifier to parse. - if "/" not in owner_prompt: - return "-", owner_prompt, commit + Returns: + Tuple[str, str, str]: A tuple containing (owner, name, commit). - owner, prompt = owner_prompt.split("/", 1) - return owner, prompt, commit + Raises: + ValueError: If the identifier doesn't match the expected formats. + """ + if ( + not identifier + or identifier.count("/") > 1 + or identifier.startswith("/") + or identifier.endswith("/") + ): + raise ValueError(f"Invalid identifier format: {identifier}") + + parts = identifier.split(":", 1) + owner_name = parts[0] + commit = parts[1] if len(parts) > 1 else "latest" + + if "/" in owner_name: + owner, name = owner_name.split("/", 1) + if not owner or not name: + raise ValueError(f"Invalid identifier format: {identifier}") + return owner, name, commit + else: + if not owner_name: + raise ValueError(f"Invalid identifier format: {identifier}") + return "-", owner_name, commit diff --git a/python/langsmith/wrappers/_openai.py b/python/langsmith/wrappers/_openai.py index 5b6798e8d..45aec0932 100644 --- a/python/langsmith/wrappers/_openai.py +++ b/python/langsmith/wrappers/_openai.py @@ -114,13 +114,13 @@ def _reduce_choices(choices: List[Choice]) -> dict: "arguments": "", } if chunk.function.name: - message["tool_calls"][index]["function"][ - "name" - ] += chunk.function.name + message["tool_calls"][index]["function"]["name"] += ( + chunk.function.name + ) if chunk.function.arguments: - message["tool_calls"][index]["function"][ - "arguments" - ] += chunk.function.arguments + message["tool_calls"][index]["function"]["arguments"] += ( + chunk.function.arguments + ) return { "index": choices[0].index, "finish_reason": next( diff --git a/python/tests/prompts/test_prompts.py b/python/tests/prompts/test_prompts.py index 4554314f4..2e6f1cb89 100644 --- a/python/tests/prompts/test_prompts.py +++ b/python/tests/prompts/test_prompts.py @@ -1,9 +1,11 @@ -import pytest from uuid import uuid4 -from langsmith.client import Client -from langsmith.schemas import Prompt, ListPromptsResponse + +import pytest from langchain_core.prompts import ChatPromptTemplate +from langsmith.client import Client +from langsmith.schemas import ListPromptsResponse, Prompt, PromptManifest + @pytest.fixture def langsmith_client() -> Client: @@ -25,68 +27,151 @@ def prompt_template_2() -> ChatPromptTemplate: ) +def test_current_tenant_is_owner(langsmith_client: Client): + settings = langsmith_client.get_settings() + assert langsmith_client.current_tenant_is_owner(settings["tenant_handle"]) + assert langsmith_client.current_tenant_is_owner("-") + assert not langsmith_client.current_tenant_is_owner("non_existent_owner") + + def test_list_prompts(langsmith_client: Client): - # Test listing prompts response = langsmith_client.list_prompts(limit=10, offset=0) assert isinstance(response, ListPromptsResponse) assert len(response.repos) <= 10 def test_get_prompt(langsmith_client: Client, prompt_template_1: ChatPromptTemplate): - # First, create a prompt to test with prompt_name = f"test_prompt_{uuid4().hex[:8]}" langsmith_client.push_prompt(prompt_name, prompt_template_1) - # Now test getting the prompt prompt = langsmith_client.get_prompt(prompt_name) assert isinstance(prompt, Prompt) assert prompt.repo_handle == prompt_name - # Clean up langsmith_client.delete_prompt(prompt_name) - assert not langsmith_client.prompt_exists(prompt_name) def test_prompt_exists(langsmith_client: Client, prompt_template_2: ChatPromptTemplate): - # Test with a non-existent prompt non_existent_prompt = f"non_existent_{uuid4().hex[:8]}" assert not langsmith_client.prompt_exists(non_existent_prompt) - # Create a prompt and test again existent_prompt = f"existent_{uuid4().hex[:8]}" langsmith_client.push_prompt(existent_prompt, prompt_template_2) assert langsmith_client.prompt_exists(existent_prompt) - # Clean up langsmith_client.delete_prompt(existent_prompt) - assert not langsmith_client.prompt_exists(existent_prompt) -def test_push_and_pull_prompt(langsmith_client: Client, prompt_template_2: ChatPromptTemplate): +def test_update_prompt(langsmith_client: Client, prompt_template_1: ChatPromptTemplate): + prompt_name = f"test_prompt_{uuid4().hex[:8]}" + langsmith_client.push_prompt(prompt_name, prompt_template_1) + + updated_data = langsmith_client.update_prompt( + prompt_name, + description="Updated description", + is_public=True, + tags=["test", "update"], + ) + assert isinstance(updated_data, dict) + + updated_prompt = langsmith_client.get_prompt(prompt_name) + assert updated_prompt.description == "Updated description" + assert updated_prompt.is_public + assert set(updated_prompt.tags) == set(["test", "update"]) + + langsmith_client.delete_prompt(prompt_name) + + +def test_delete_prompt(langsmith_client: Client, prompt_template_1: ChatPromptTemplate): + prompt_name = f"test_prompt_{uuid4().hex[:8]}" + langsmith_client.push_prompt(prompt_name, prompt_template_1) + + assert langsmith_client.prompt_exists(prompt_name) + langsmith_client.delete_prompt(prompt_name) + assert not langsmith_client.prompt_exists(prompt_name) + + +def test_pull_prompt_manifest( + langsmith_client: Client, prompt_template_1: ChatPromptTemplate +): + prompt_name = f"test_prompt_{uuid4().hex[:8]}" + langsmith_client.push_prompt(prompt_name, prompt_template_1) + + manifest = langsmith_client.pull_prompt_manifest(prompt_name) + assert isinstance(manifest, PromptManifest) + assert manifest.repo == prompt_name + + langsmith_client.delete_prompt(prompt_name) + + +def test_pull_prompt(langsmith_client: Client, prompt_template_1: ChatPromptTemplate): + prompt_name = f"test_prompt_{uuid4().hex[:8]}" + langsmith_client.push_prompt(prompt_name, prompt_template_1) + + pulled_prompt = langsmith_client.pull_prompt(prompt_name) + assert isinstance(pulled_prompt, ChatPromptTemplate) + + langsmith_client.delete_prompt(prompt_name) + + +def test_push_and_pull_prompt( + langsmith_client: Client, prompt_template_2: ChatPromptTemplate +): prompt_name = f"test_prompt_{uuid4().hex[:8]}" - # Test pushing a prompt push_result = langsmith_client.push_prompt(prompt_name, prompt_template_2) - assert isinstance(push_result, str) # Should return a URL + assert isinstance(push_result, str) - # Test pulling the prompt - langsmith_client.pull_prompt(prompt_name) + pulled_prompt = langsmith_client.pull_prompt(prompt_name) + assert isinstance(pulled_prompt, ChatPromptTemplate) - # Clean up langsmith_client.delete_prompt(prompt_name) + # should fail + with pytest.raises(ValueError): + langsmith_client.push_prompt(f"random_handle/{prompt_name}", prompt_template_2) + -def test_push_prompt_manifest(langsmith_client: Client, prompt_template_2: ChatPromptTemplate): +def test_push_prompt_manifest( + langsmith_client: Client, prompt_template_2: ChatPromptTemplate +): prompt_name = f"test_prompt_manifest_{uuid4().hex[:8]}" - # Test pushing a prompt manifest result = langsmith_client.push_prompt_manifest(prompt_name, prompt_template_2) - assert isinstance(result, str) # Should return a URL + assert isinstance(result, str) - # Verify the pushed manifest pulled_prompt_manifest = langsmith_client.pull_prompt_manifest(prompt_name) latest_commit_hash = langsmith_client._get_latest_commit_hash(f"-/{prompt_name}") assert pulled_prompt_manifest.commit_hash == latest_commit_hash - # Clean up + langsmith_client.delete_prompt(prompt_name) + + +def test_like_unlike_prompt( + langsmith_client: Client, prompt_template_1: ChatPromptTemplate +): + prompt_name = f"test_prompt_{uuid4().hex[:8]}" + langsmith_client.push_prompt(prompt_name, prompt_template_1) + + langsmith_client.like_prompt(prompt_name) + prompt = langsmith_client.get_prompt(prompt_name) + assert prompt.num_likes == 1 + + langsmith_client.unlike_prompt(prompt_name) + prompt = langsmith_client.get_prompt(prompt_name) + assert prompt.num_likes == 0 + + langsmith_client.delete_prompt(prompt_name) + + +def test_get_latest_commit_hash( + langsmith_client: Client, prompt_template_1: ChatPromptTemplate +): + prompt_name = f"test_prompt_{uuid4().hex[:8]}" + langsmith_client.push_prompt(prompt_name, prompt_template_1) + + commit_hash = langsmith_client._get_latest_commit_hash(f"-/{prompt_name}") + assert isinstance(commit_hash, str) + assert len(commit_hash) > 0 + langsmith_client.delete_prompt(prompt_name) diff --git a/python/tests/unit_tests/test_utils.py b/python/tests/unit_tests/test_utils.py index 9cadaa9cb..bd57385cf 100644 --- a/python/tests/unit_tests/test_utils.py +++ b/python/tests/unit_tests/test_utils.py @@ -13,7 +13,6 @@ import attr import dataclasses_json import pytest -from pydantic import BaseModel import langsmith.utils as ls_utils from langsmith import Client, traceable @@ -163,19 +162,6 @@ def __init__(self) -> None: class MyClassWithSlots: __slots__ = ["x", "y"] - - def __init__(self, x: int) -> None: - self.x = x - self.y = "y" - - class MyPydantic(BaseModel): - foo: str - bar: int - baz: dict - - @dataclasses.dataclass - class MyDataclass: - foo: str bar: int def something(self) -> None: @@ -264,3 +250,50 @@ class MyNamedTuple(NamedTuple): "fake_json": ClassWithFakeJson(), } assert ls_utils.deepish_copy(my_dict) == my_dict + + +def test_is_version_greater_or_equal(): + # Test versions equal to 0.5.23 + assert ls_utils.is_version_greater_or_equal("0.5.23", "0.5.23") + + # Test versions greater than 0.5.23 + assert ls_utils.is_version_greater_or_equal("0.5.24", "0.5.23") + assert ls_utils.is_version_greater_or_equal("0.6.0", "0.5.23") + assert ls_utils.is_version_greater_or_equal("1.0.0", "0.5.23") + + # Test versions less than 0.5.23 + assert not ls_utils.is_version_greater_or_equal("0.5.22", "0.5.23") + assert not ls_utils.is_version_greater_or_equal("0.5.0", "0.5.23") + assert not ls_utils.is_version_greater_or_equal("0.4.99", "0.5.23") + + +def test_parse_prompt_identifier(): + # Valid cases + assert ls_utils.parse_prompt_identifier("name") == ("-", "name", "latest") + assert ls_utils.parse_prompt_identifier("owner/name") == ("owner", "name", "latest") + assert ls_utils.parse_prompt_identifier("owner/name:commit") == ( + "owner", + "name", + "commit", + ) + assert ls_utils.parse_prompt_identifier("name:commit") == ("-", "name", "commit") + + # Invalid cases + invalid_identifiers = [ + "", + "/", + ":", + "owner/", + "/name", + "owner//name", + "owner/name/", + "owner/name/extra", + ":commit", + ] + + for invalid_id in invalid_identifiers: + try: + ls_utils.parse_prompt_identifier(invalid_id) + assert False, f"Expected ValueError for identifier: {invalid_id}" + except ValueError: + pass # This is the expected behavior From 124b78860342cb1d1dd819774b579ede2ed5e88f Mon Sep 17 00:00:00 2001 From: Maddy Adams Date: Thu, 11 Jul 2024 19:27:45 -0700 Subject: [PATCH 10/61] fix --- python/tests/unit_tests/test_utils.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/python/tests/unit_tests/test_utils.py b/python/tests/unit_tests/test_utils.py index bd57385cf..09af201a7 100644 --- a/python/tests/unit_tests/test_utils.py +++ b/python/tests/unit_tests/test_utils.py @@ -13,6 +13,7 @@ import attr import dataclasses_json import pytest +from pydantic import BaseModel import langsmith.utils as ls_utils from langsmith import Client, traceable @@ -162,6 +163,18 @@ def __init__(self) -> None: class MyClassWithSlots: __slots__ = ["x", "y"] + def __init__(self, x: int) -> None: + self.x = x + self.y = "y" + + class MyPydantic(BaseModel): + foo: str + bar: int + baz: dict + + @dataclasses.dataclass + class MyDataclass: + foo: str bar: int def something(self) -> None: From df4f726d33c3ee495819dff44b160503e77c953b Mon Sep 17 00:00:00 2001 From: Maddy Adams Date: Thu, 11 Jul 2024 20:27:45 -0700 Subject: [PATCH 11/61] format --- python/langsmith/client.py | 90 ++++++++++++--------------- python/langsmith/utils.py | 4 +- python/tests/prompts/test_prompts.py | 15 ----- python/tests/unit_tests/test_utils.py | 1 + 4 files changed, 44 insertions(+), 66 deletions(-) diff --git a/python/langsmith/client.py b/python/langsmith/client.py index 86a13b107..c89c1e777 100644 --- a/python/langsmith/client.py +++ b/python/langsmith/client.py @@ -4630,7 +4630,8 @@ def _like_or_unlike_prompt( A dictionary with the key 'likes' and the count of likes as the value. Raises: - requests.exceptions.HTTPError: If the prompt is not found or another error occurs. + requests.exceptions.HTTPError: If the prompt is not found or + another error occurs. """ owner, prompt_name, _ = ls_utils.parse_prompt_identifier(prompt_identifier) response = self.request_with_retries( @@ -4673,7 +4674,8 @@ def list_prompts( offset (int): The number of prompts to skip. Defaults to 0. Returns: - ls_schemas.ListPromptsResponse: A response object containing the list of prompts. + ls_schemas.ListPromptsResponse: A response object containing + the list of prompts. """ params = {"limit": limit, "offset": offset} response = self.request_with_retries("GET", "/repos", params=params) @@ -4683,13 +4685,15 @@ def get_prompt(self, prompt_identifier: str) -> ls_schemas.Prompt: """Get a specific prompt by its identifier. Args: - prompt_identifier (str): The identifier of the prompt. The identifier should be in the format "prompt_name" or "owner/prompt_name". + prompt_identifier (str): The identifier of the prompt. + The identifier should be in the format "prompt_name" or "owner/prompt_name". Returns: ls_schemas.Prompt: The prompt object. Raises: - requests.exceptions.HTTPError: If the prompt is not found or another error occurs. + requests.exceptions.HTTPError: If the prompt is not found or + another error occurs. """ owner, prompt_name, _ = ls_utils.parse_prompt_identifier(prompt_identifier) response = self.request_with_retries( @@ -4752,7 +4756,9 @@ def delete_prompt(self, prompt_identifier: str) -> bool: owner, prompt_name, _ = ls_utils.parse_prompt_identifier(prompt_identifier) if not self.current_tenant_is_owner(owner): raise ValueError( - f"Cannot delete prompt for another tenant. Current tenant: {self.get_settings()['tenant_handle']}, Requested tenant: {owner}" + f"Cannot delete prompt for another tenant.\n" + f"Current tenant: {self.get_settings()['tenant_handle']},\n" + f"Requested tenant: {owner}" ) response = self.request_with_retries("DELETE", f"/repos/{owner}/{prompt_name}") return response.status_code == 204 @@ -4789,32 +4795,35 @@ def pull_prompt_manifest(self, prompt_identifier: str) -> ls_schemas.PromptManif ) def pull_prompt(self, prompt_identifier: str) -> Any: - """Pull a prompt and return it as a LangChain object. + """Pull a prompt and return it as a LangChain PromptTemplate. + + This method requires `langchain_core` to convert the prompt manifest. Args: prompt_identifier (str): The identifier of the prompt. Returns: - Any: The prompt object. + Any: The prompt object in the specified format. """ from langchain_core.load.load import loads from langchain_core.prompts import BasePromptTemplate - response = self.pull_prompt_manifest(prompt_identifier) - obj = loads(json.dumps(response.manifest)) - if isinstance(obj, BasePromptTemplate): - if obj.metadata is None: - obj.metadata = {} - obj.metadata.update( + prompt_manifest = self.pull_prompt_manifest(prompt_identifier) + prompt = loads(json.dumps(prompt_manifest.manifest)) + if isinstance(prompt, BasePromptTemplate): + if prompt.metadata is None: + prompt.metadata = {} + prompt.metadata.update( { - "lc_hub_owner": response.owner, - "lc_hub_repo": response.repo, - "lc_hub_commit_hash": response.commit_hash, + "lc_hub_owner": prompt_manifest.owner, + "lc_hub_repo": prompt_manifest.repo, + "lc_hub_commit_hash": prompt_manifest.commit_hash, } ) - return obj - def push_prompt_manifest( + return prompt + + def push_prompt( self, prompt_identifier: str, manifest_json: Any, @@ -4827,7 +4836,8 @@ def push_prompt_manifest( Args: prompt_identifier (str): The identifier of the prompt. manifest_json (Any): The manifest to push. - parent_commit_hash (Optional[str]): The parent commit hash. Defaults to "latest". + parent_commit_hash (Optional[str]): The parent commit hash. + Defaults to "latest". is_public (bool): Whether the prompt should be public. Defaults to False. description (str): A description of the prompt. Defaults to an empty string. @@ -4835,7 +4845,8 @@ def push_prompt_manifest( str: The URL of the pushed prompt. Raises: - ValueError: If a public prompt is attempted without a tenant handle or if the current tenant is not the owner. + ValueError: If a public prompt is attempted without a tenant handle or + if the current tenant is not the owner. """ from langchain_core.load.dump import dumps @@ -4844,8 +4855,10 @@ def push_prompt_manifest( if is_public and not settings.get("tenant_handle"): raise ValueError( - "Cannot create a public prompt without first creating a LangChain Hub handle. " - "You can add a handle by creating a public prompt at: https://smith.langchain.com/prompts" + "Cannot create a public prompt without first\n" + "creating a LangChain Hub handle. " + "You can add a handle by creating a public prompt at:\n" + "https://smith.langchain.com/prompts" ) owner, prompt_name, _ = ls_utils.parse_prompt_identifier(prompt_identifier) @@ -4853,7 +4866,9 @@ def push_prompt_manifest( if not self.current_tenant_is_owner(owner): raise ValueError( - f"Cannot create prompt for another tenant. Current tenant: {settings['tenant_handle'] or 'no handle'}, Requested tenant: {owner}" + "Cannot create prompt for another tenant." + f"Current tenant: {settings['tenant_handle'] or 'no handle'}" + f", Requested tenant: {owner}" ) if not self.prompt_exists(prompt_full_name): @@ -4878,32 +4893,9 @@ def push_prompt_manifest( commit_hash = response.json()["commit"]["commit_hash"] short_hash = commit_hash[:8] - return f"{self._host_url}/prompts/{prompt_name}/{short_hash}?organizationId={settings['id']}" - - def push_prompt( - self, - prompt_identifier: str, - obj: Any, - parent_commit_hash: Optional[str] = "latest", - is_public: bool = False, - description: str = "", - ) -> str: - """Push a prompt object to the LangSmith API. - - This method is a wrapper around push_prompt_manifest. - - Args: - prompt_identifier (str): The identifier of the prompt. The format is "name" or "-/name" or "workspace_handle/name". - obj (Any): The prompt object to push. - parent_commit_hash (Optional[str]): The parent commit hash. Defaults to "latest". - is_public (bool): Whether the prompt should be public. Defaults to False. - description (str): A description of the prompt. Defaults to an empty string. - - Returns: - str: The URL of the pushed prompt. - """ - return self.push_prompt_manifest( - prompt_identifier, obj, parent_commit_hash, is_public, description + return ( + f"{self._host_url}/prompts/{prompt_name}/{short_hash}" + f"?organizationId={settings['id']}" ) diff --git a/python/langsmith/utils.py b/python/langsmith/utils.py index 6fb0f0ff9..9f20ffd8c 100644 --- a/python/langsmith/utils.py +++ b/python/langsmith/utils.py @@ -573,13 +573,13 @@ def is_version_greater_or_equal(current_version, target_version): def parse_prompt_identifier(identifier: str) -> Tuple[str, str, str]: - """Parse a string in the format of `owner/name[:commit]` or `name[:commit]` and returns a tuple of (owner, name, commit). + """Parse a string in the format of owner/name:hash, name:hash, owner/name, or name. Args: identifier (str): The prompt identifier to parse. Returns: - Tuple[str, str, str]: A tuple containing (owner, name, commit). + Tuple[str, str, str]: A tuple containing (owner, name, hash). Raises: ValueError: If the identifier doesn't match the expected formats. diff --git a/python/tests/prompts/test_prompts.py b/python/tests/prompts/test_prompts.py index 2e6f1cb89..ec7f76da1 100644 --- a/python/tests/prompts/test_prompts.py +++ b/python/tests/prompts/test_prompts.py @@ -132,21 +132,6 @@ def test_push_and_pull_prompt( langsmith_client.push_prompt(f"random_handle/{prompt_name}", prompt_template_2) -def test_push_prompt_manifest( - langsmith_client: Client, prompt_template_2: ChatPromptTemplate -): - prompt_name = f"test_prompt_manifest_{uuid4().hex[:8]}" - - result = langsmith_client.push_prompt_manifest(prompt_name, prompt_template_2) - assert isinstance(result, str) - - pulled_prompt_manifest = langsmith_client.pull_prompt_manifest(prompt_name) - latest_commit_hash = langsmith_client._get_latest_commit_hash(f"-/{prompt_name}") - assert pulled_prompt_manifest.commit_hash == latest_commit_hash - - langsmith_client.delete_prompt(prompt_name) - - def test_like_unlike_prompt( langsmith_client: Client, prompt_template_1: ChatPromptTemplate ): diff --git a/python/tests/unit_tests/test_utils.py b/python/tests/unit_tests/test_utils.py index 09af201a7..8fd493478 100644 --- a/python/tests/unit_tests/test_utils.py +++ b/python/tests/unit_tests/test_utils.py @@ -163,6 +163,7 @@ def __init__(self) -> None: class MyClassWithSlots: __slots__ = ["x", "y"] + def __init__(self, x: int) -> None: self.x = x self.y = "y" From c0eeb92640b47726d0aada7264423bea93e557fe Mon Sep 17 00:00:00 2001 From: Maddy Adams Date: Fri, 12 Jul 2024 10:21:19 -0700 Subject: [PATCH 12/61] expand list_prompts functionality --- _scripts/_fetch_schema.py | 18 ++++++++++++------ python/langsmith/client.py | 32 ++++++++++++++++++++++++++++++-- python/langsmith/schemas.py | 13 +++++++++++++ 3 files changed, 55 insertions(+), 8 deletions(-) diff --git a/_scripts/_fetch_schema.py b/_scripts/_fetch_schema.py index 741e12a9c..ba8c171bd 100644 --- a/_scripts/_fetch_schema.py +++ b/_scripts/_fetch_schema.py @@ -1,4 +1,5 @@ """Fetch and prune the Langsmith spec.""" + import argparse from pathlib import Path @@ -19,7 +20,9 @@ def process_schema(sub_schema): get_dependencies(schema, sub_schema["$ref"].split("/")[-1], new_components) else: if "items" in sub_schema and "$ref" in sub_schema["items"]: - get_dependencies(schema, sub_schema["items"]["$ref"].split("/")[-1], new_components) + get_dependencies( + schema, sub_schema["items"]["$ref"].split("/")[-1], new_components + ) for keyword in ["anyOf", "oneOf", "allOf"]: if keyword in sub_schema: for item in sub_schema[keyword]: @@ -38,8 +41,6 @@ def process_schema(sub_schema): process_schema(item) - - def _extract_langsmith_routes_and_properties(schema, operation_ids): new_paths = {} new_components = {"schemas": {}} @@ -98,20 +99,25 @@ def test_openapi_specification(spec: dict): assert errors is None, f"OpenAPI validation failed: {errors}" -def main(out_file: str = "openapi.yaml", url: str = "https://web.smith.langchain.com/openapi.json"): +def main( + out_file: str = "openapi.yaml", + url: str = "https://web.smith.langchain.com/openapi.json", +): langsmith_schema = get_langsmith_runs_schema(url=url) parent_dir = Path(__file__).parent.parent test_openapi_specification(langsmith_schema) with (parent_dir / "openapi" / out_file).open("w") as f: # Sort the schema keys so the openapi version and info come at the top - for key in ['openapi', 'info', 'paths', 'components']: + for key in ["openapi", "info", "paths", "components"]: langsmith_schema[key] = langsmith_schema.pop(key) f.write(yaml.dump(langsmith_schema, sort_keys=False)) if __name__ == "__main__": parser = argparse.ArgumentParser() - parser.add_argument("--url", type=str, default="https://web.smith.langchain.com/openapi.json") + parser.add_argument( + "--url", type=str, default="https://web.smith.langchain.com/openapi.json" + ) parser.add_argument("--output", type=str, default="openapi.yaml") args = parser.parse_args() main(args.output, url=args.url) diff --git a/python/langsmith/client.py b/python/langsmith/client.py index c89c1e777..503964002 100644 --- a/python/langsmith/client.py +++ b/python/langsmith/client.py @@ -4665,19 +4665,47 @@ def unlike_prompt(self, prompt_identifier: str) -> Dict[str, int]: return self._like_or_unlike_prompt(prompt_identifier, like=False) def list_prompts( - self, limit: int = 100, offset: int = 0 + self, + *, + limit: int = 100, + offset: int = 0, + is_public: Optional[bool] = None, + is_archived: Optional[bool] = False, + sort_field: ls_schemas.PromptsSortField = ls_schemas.PromptsSortField.updated_at, + sort_direction: Literal["desc", "asc"] = "desc", + query: Optional[str] = None, ) -> ls_schemas.ListPromptsResponse: """List prompts with pagination. Args: limit (int): The maximum number of prompts to return. Defaults to 100. offset (int): The number of prompts to skip. Defaults to 0. + is_public (Optional[bool]): Filter prompts by if they are public. + is_archived (Optional[bool]): Filter prompts by if they are archived. + sort_field (ls_schemas.PromptsSortField): The field to sort by. + Defaults to "updated_at". + sort_direction (Literal["desc", "asc"]): The order to sort by. Defaults to "desc". + query (Optional[str]): Filter prompts by a search query. Returns: ls_schemas.ListPromptsResponse: A response object containing the list of prompts. """ - params = {"limit": limit, "offset": offset} + params = { + "limit": limit, + "offset": offset, + "is_public": "true" + if is_public + else "false" + if is_public is not None + else None, + "is_archived": "true" if is_archived else "false", + "sort_field": sort_field, + "sort_direction": sort_direction, + "query": query, + "match_prefix": "true" if query else None, + } + response = self.request_with_retries("GET", "/repos", params=params) return ls_schemas.ListPromptsResponse(**response.json()) diff --git a/python/langsmith/schemas.py b/python/langsmith/schemas.py index ebebfcb63..8166923a4 100644 --- a/python/langsmith/schemas.py +++ b/python/langsmith/schemas.py @@ -824,3 +824,16 @@ class ListPromptsResponse(BaseModel): """The list of prompts.""" total: int """The total number of prompts.""" + + +class PromptsSortField(str, Enum): + """Enum for sorting fields for prompts.""" + + num_downloads = "num_downloads" + """Number of downloads.""" + num_views = "num_views" + """Number of views.""" + updated_at = "updated_at" + """Last updated time.""" + num_likes = "num_likes" + """Number of likes.""" From 910fc028c7abd764c4a357758d6b67bd8694ba21 Mon Sep 17 00:00:00 2001 From: Maddy Adams Date: Fri, 12 Jul 2024 10:28:47 -0700 Subject: [PATCH 13/61] line length --- python/langsmith/client.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/python/langsmith/client.py b/python/langsmith/client.py index 503964002..d272d4c3c 100644 --- a/python/langsmith/client.py +++ b/python/langsmith/client.py @@ -4671,7 +4671,8 @@ def list_prompts( offset: int = 0, is_public: Optional[bool] = None, is_archived: Optional[bool] = False, - sort_field: ls_schemas.PromptsSortField = ls_schemas.PromptsSortField.updated_at, + sort_field: ls_schemas.PromptsSortField = + ls_schemas.PromptsSortField.updated_at, sort_direction: Literal["desc", "asc"] = "desc", query: Optional[str] = None, ) -> ls_schemas.ListPromptsResponse: @@ -4684,7 +4685,8 @@ def list_prompts( is_archived (Optional[bool]): Filter prompts by if they are archived. sort_field (ls_schemas.PromptsSortField): The field to sort by. Defaults to "updated_at". - sort_direction (Literal["desc", "asc"]): The order to sort by. Defaults to "desc". + sort_direction (Literal["desc", "asc"]): The order to sort by. + Defaults to "desc". query (Optional[str]): Filter prompts by a search query. Returns: From 19e663e168dea75efc0960ea5511f9d940db8342 Mon Sep 17 00:00:00 2001 From: Maddy Adams Date: Fri, 12 Jul 2024 11:07:18 -0700 Subject: [PATCH 14/61] feat: methods to convert prompt to openai and anthropic formats --- python/langsmith/client.py | 58 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 56 insertions(+), 2 deletions(-) diff --git a/python/langsmith/client.py b/python/langsmith/client.py index d272d4c3c..c11cbc980 100644 --- a/python/langsmith/client.py +++ b/python/langsmith/client.py @@ -4671,8 +4671,7 @@ def list_prompts( offset: int = 0, is_public: Optional[bool] = None, is_archived: Optional[bool] = False, - sort_field: ls_schemas.PromptsSortField = - ls_schemas.PromptsSortField.updated_at, + sort_field: ls_schemas.PromptsSortField = ls_schemas.PromptsSortField.updated_at, sort_direction: Literal["desc", "asc"] = "desc", query: Optional[str] = None, ) -> ls_schemas.ListPromptsResponse: @@ -4928,6 +4927,61 @@ def push_prompt( f"?organizationId={settings['id']}" ) + def convert_to_openai_format( + self, messages: Any, stop: Optional[List[str]] = None, **kwargs: Any + ) -> dict: + """Convert a prompt to OpenAI format. + + Requires the `langchain_openai` package to be installed. + + Args: + messages (Any): The messages to convert. + stop (Optional[List[str]]): Stop sequences for the prompt. + **kwargs: Additional arguments for the conversion. + + Returns: + dict: The prompt in OpenAI format. + """ + from langchain_openai import ChatOpenAI + + openai = ChatOpenAI() + + try: + return openai._get_request_payload(messages, stop=stop, **kwargs) + except Exception as e: + print(e) + return None + + def convert_to_anthropic_format( + self, + messages: Any, + model_name: Optional[str] = "claude-2", + stop: Optional[List[str]] = None, + **kwargs: Any, + ) -> dict: + """Convert a prompt to Anthropic format. + + Requires the `langchain_anthropic` package to be installed. + + Args: + messages (Any): The messages to convert. + model_name (Optional[str]): The model name to use. Defaults to "claude-2". + stop (Optional[List[str]]): Stop sequences for the prompt. + **kwargs: Additional arguments for the conversion. + + Returns: + dict: The prompt in Anthropic format. + """ + from langchain_anthropic import ChatAnthropic + + anthropic = ChatAnthropic(model_name=model_name) + + try: + return anthropic._get_request_payload(messages, stop=stop, **kwargs) + except Exception as e: + print(e) + return None + def _tracing_thread_drain_queue( tracing_queue: Queue, limit: int = 100, block: bool = True From 3ee0beb93e2fd29a40dac456164582f293b02bc9 Mon Sep 17 00:00:00 2001 From: Maddy Adams Date: Fri, 12 Jul 2024 11:16:34 -0700 Subject: [PATCH 15/61] integration tests --- python/langsmith/schemas.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/python/langsmith/schemas.py b/python/langsmith/schemas.py index 8166923a4..8bb542ad6 100644 --- a/python/langsmith/schemas.py +++ b/python/langsmith/schemas.py @@ -777,9 +777,9 @@ class Prompt(BaseModel): """The name of the prompt.""" full_name: str """The full name of the prompt. (owner + repo_handle)""" - description: str | None + description: str = None """The description of the prompt.""" - readme: str | None + readme: str = None """The README of the prompt.""" id: str """The ID of the prompt.""" @@ -795,9 +795,9 @@ class Prompt(BaseModel): """Whether the prompt is archived.""" tags: List[str] """The tags associated with the prompt.""" - original_repo_id: str | None + original_repo_id: str = None """The ID of the original prompt, if forked.""" - upstream_repo_id: str | None + upstream_repo_id: str = None """The ID of the upstream prompt, if forked.""" num_likes: int """The number of likes.""" @@ -807,13 +807,13 @@ class Prompt(BaseModel): """The number of views.""" liked_by_auth_user: bool """Whether the prompt is liked by the authenticated user.""" - last_commit_hash: str | None + last_commit_hash: str = None """The hash of the last commit.""" num_commits: int """The number of commits.""" - original_repo_full_name: str | None + original_repo_full_name: str = None """The full name of the original prompt, if forked.""" - upstream_repo_full_name: str | None + upstream_repo_full_name: str = None """The full name of the upstream prompt, if forked.""" From b175603f8ca51b9bbe1dec40e1e6040c6aab3900 Mon Sep 17 00:00:00 2001 From: Maddy Adams Date: Fri, 12 Jul 2024 11:19:02 -0700 Subject: [PATCH 16/61] format --- python/langsmith/client.py | 11 ++++------- python/langsmith/wrappers/_openai.py | 12 ++++++------ 2 files changed, 10 insertions(+), 13 deletions(-) diff --git a/python/langsmith/client.py b/python/langsmith/client.py index d272d4c3c..469b13634 100644 --- a/python/langsmith/client.py +++ b/python/langsmith/client.py @@ -4671,8 +4671,7 @@ def list_prompts( offset: int = 0, is_public: Optional[bool] = None, is_archived: Optional[bool] = False, - sort_field: ls_schemas.PromptsSortField = - ls_schemas.PromptsSortField.updated_at, + sort_field: ls_schemas.PromptsSortField = "updated_at", sort_direction: Literal["desc", "asc"] = "desc", query: Optional[str] = None, ) -> ls_schemas.ListPromptsResponse: @@ -4696,11 +4695,9 @@ def list_prompts( params = { "limit": limit, "offset": offset, - "is_public": "true" - if is_public - else "false" - if is_public is not None - else None, + "is_public": ( + "true" if is_public else "false" if is_public is not None else None + ), "is_archived": "true" if is_archived else "false", "sort_field": sort_field, "sort_direction": sort_direction, diff --git a/python/langsmith/wrappers/_openai.py b/python/langsmith/wrappers/_openai.py index 45aec0932..5b6798e8d 100644 --- a/python/langsmith/wrappers/_openai.py +++ b/python/langsmith/wrappers/_openai.py @@ -114,13 +114,13 @@ def _reduce_choices(choices: List[Choice]) -> dict: "arguments": "", } if chunk.function.name: - message["tool_calls"][index]["function"]["name"] += ( - chunk.function.name - ) + message["tool_calls"][index]["function"][ + "name" + ] += chunk.function.name if chunk.function.arguments: - message["tool_calls"][index]["function"]["arguments"] += ( - chunk.function.arguments - ) + message["tool_calls"][index]["function"][ + "arguments" + ] += chunk.function.arguments return { "index": choices[0].index, "finish_reason": next( From fd9aaa2e061ff701e85d1e5f6234ae3a88a66824 Mon Sep 17 00:00:00 2001 From: Maddy Adams Date: Fri, 12 Jul 2024 11:26:13 -0700 Subject: [PATCH 17/61] schema --- python/langsmith/schemas.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/python/langsmith/schemas.py b/python/langsmith/schemas.py index 8bb542ad6..38343320f 100644 --- a/python/langsmith/schemas.py +++ b/python/langsmith/schemas.py @@ -771,15 +771,11 @@ class PromptManifest(BaseModel): class Prompt(BaseModel): """Represents a Prompt with metadata.""" - owner: str - """The handle of the owner of the prompt.""" repo_handle: str """The name of the prompt.""" - full_name: str - """The full name of the prompt. (owner + repo_handle)""" - description: str = None + description: Optional[str] = None """The description of the prompt.""" - readme: str = None + readme: Optional[str] = None """The README of the prompt.""" id: str """The ID of the prompt.""" @@ -795,10 +791,14 @@ class Prompt(BaseModel): """Whether the prompt is archived.""" tags: List[str] """The tags associated with the prompt.""" - original_repo_id: str = None + original_repo_id: Optional[str] = None """The ID of the original prompt, if forked.""" upstream_repo_id: str = None """The ID of the upstream prompt, if forked.""" + owner: Optional[str] + """The handle of the owner of the prompt.""" + full_name: str + """The full name of the prompt. (owner + repo_handle)""" num_likes: int """The number of likes.""" num_downloads: int @@ -807,7 +807,7 @@ class Prompt(BaseModel): """The number of views.""" liked_by_auth_user: bool """Whether the prompt is liked by the authenticated user.""" - last_commit_hash: str = None + last_commit_hash: Optional[str] = None """The hash of the last commit.""" num_commits: int """The number of commits.""" From 5df0391527d77646c377ca005e6c013bcd708f0a Mon Sep 17 00:00:00 2001 From: Maddy Adams Date: Fri, 12 Jul 2024 11:32:13 -0700 Subject: [PATCH 18/61] more fixes --- python/langsmith/client.py | 2 +- python/langsmith/schemas.py | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/python/langsmith/client.py b/python/langsmith/client.py index 469b13634..88e189540 100644 --- a/python/langsmith/client.py +++ b/python/langsmith/client.py @@ -4671,7 +4671,7 @@ def list_prompts( offset: int = 0, is_public: Optional[bool] = None, is_archived: Optional[bool] = False, - sort_field: ls_schemas.PromptsSortField = "updated_at", + sort_field: ls_schemas.PromptSortField = ls_schemas.PromptSortField.updated_at, sort_direction: Literal["desc", "asc"] = "desc", query: Optional[str] = None, ) -> ls_schemas.ListPromptsResponse: diff --git a/python/langsmith/schemas.py b/python/langsmith/schemas.py index 38343320f..c37b9d14c 100644 --- a/python/langsmith/schemas.py +++ b/python/langsmith/schemas.py @@ -793,7 +793,7 @@ class Prompt(BaseModel): """The tags associated with the prompt.""" original_repo_id: Optional[str] = None """The ID of the original prompt, if forked.""" - upstream_repo_id: str = None + upstream_repo_id: Optional[str] = None """The ID of the upstream prompt, if forked.""" owner: Optional[str] """The handle of the owner of the prompt.""" @@ -811,9 +811,9 @@ class Prompt(BaseModel): """The hash of the last commit.""" num_commits: int """The number of commits.""" - original_repo_full_name: str = None + original_repo_full_name: Optional[str] = None """The full name of the original prompt, if forked.""" - upstream_repo_full_name: str = None + upstream_repo_full_name: Optional[str] = None """The full name of the upstream prompt, if forked.""" @@ -826,7 +826,7 @@ class ListPromptsResponse(BaseModel): """The total number of prompts.""" -class PromptsSortField(str, Enum): +class PromptSortField(str, Enum): """Enum for sorting fields for prompts.""" num_downloads = "num_downloads" From 89feff0962232eca0c88e991d0a48d7fa95e28c5 Mon Sep 17 00:00:00 2001 From: Maddy Adams Date: Fri, 12 Jul 2024 11:34:50 -0700 Subject: [PATCH 19/61] fix more --- python/langsmith/client.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/python/langsmith/client.py b/python/langsmith/client.py index 88e189540..2ac9a31fe 100644 --- a/python/langsmith/client.py +++ b/python/langsmith/client.py @@ -4708,7 +4708,7 @@ def list_prompts( response = self.request_with_retries("GET", "/repos", params=params) return ls_schemas.ListPromptsResponse(**response.json()) - def get_prompt(self, prompt_identifier: str) -> ls_schemas.Prompt: + def get_prompt(self, prompt_identifier: str) -> ls_schemas.Prompt | None: """Get a specific prompt by its identifier. Args: @@ -4729,6 +4729,7 @@ def get_prompt(self, prompt_identifier: str) -> ls_schemas.Prompt: if response.status_code == 200: return ls_schemas.Prompt(**response.json()["repo"]) response.raise_for_status() + return None def update_prompt( self, @@ -4809,7 +4810,7 @@ def pull_prompt_manifest(self, prompt_identifier: str) -> ls_schemas.PromptManif self.info.version, "0.5.23" ) - if not use_optimization and (commit_hash is None or commit_hash == "latest"): + if not use_optimization and commit_hash == "latest": commit_hash = self._get_latest_commit_hash(f"{owner}/{prompt_name}") if commit_hash is None: raise ValueError("No commits found") From fd61a4c5755f6d86b14b3538000f5655ebd92394 Mon Sep 17 00:00:00 2001 From: Maddy Adams Date: Fri, 12 Jul 2024 11:40:30 -0700 Subject: [PATCH 20/61] working through CI --- python/langsmith/client.py | 4 ++-- python/tests/prompts/test_prompts.py | 3 +++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/python/langsmith/client.py b/python/langsmith/client.py index 2ac9a31fe..f99a640e5 100644 --- a/python/langsmith/client.py +++ b/python/langsmith/client.py @@ -4708,7 +4708,7 @@ def list_prompts( response = self.request_with_retries("GET", "/repos", params=params) return ls_schemas.ListPromptsResponse(**response.json()) - def get_prompt(self, prompt_identifier: str) -> ls_schemas.Prompt | None: + def get_prompt(self, prompt_identifier: str) -> Optional[ls_schemas.Prompt]: """Get a specific prompt by its identifier. Args: @@ -4716,7 +4716,7 @@ def get_prompt(self, prompt_identifier: str) -> ls_schemas.Prompt | None: The identifier should be in the format "prompt_name" or "owner/prompt_name". Returns: - ls_schemas.Prompt: The prompt object. + Optional[ls_schemas.Prompt]: The prompt object. Raises: requests.exceptions.HTTPError: If the prompt is not found or diff --git a/python/tests/prompts/test_prompts.py b/python/tests/prompts/test_prompts.py index ec7f76da1..e0d77c30d 100644 --- a/python/tests/prompts/test_prompts.py +++ b/python/tests/prompts/test_prompts.py @@ -75,6 +75,7 @@ def test_update_prompt(langsmith_client: Client, prompt_template_1: ChatPromptTe assert isinstance(updated_data, dict) updated_prompt = langsmith_client.get_prompt(prompt_name) + assert isinstance(updated_prompt, Prompt) assert updated_prompt.description == "Updated description" assert updated_prompt.is_public assert set(updated_prompt.tags) == set(["test", "update"]) @@ -140,10 +141,12 @@ def test_like_unlike_prompt( langsmith_client.like_prompt(prompt_name) prompt = langsmith_client.get_prompt(prompt_name) + assert isinstance(prompt, Prompt) assert prompt.num_likes == 1 langsmith_client.unlike_prompt(prompt_name) prompt = langsmith_client.get_prompt(prompt_name) + assert isinstance(prompt, Prompt) assert prompt.num_likes == 0 langsmith_client.delete_prompt(prompt_name) From 78c688c7a6ff220c06e1a6a41665a6a5464c2763 Mon Sep 17 00:00:00 2001 From: Maddy Adams Date: Fri, 12 Jul 2024 11:44:44 -0700 Subject: [PATCH 21/61] ci --- python/langsmith/client.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/python/langsmith/client.py b/python/langsmith/client.py index f99a640e5..6c0df3bab 100644 --- a/python/langsmith/client.py +++ b/python/langsmith/client.py @@ -4638,7 +4638,7 @@ def _like_or_unlike_prompt( "POST", f"/likes/{owner}/{prompt_name}", json={"like": like} ) response.raise_for_status() - return response.json + return response.json() def like_prompt(self, prompt_identifier: str) -> Dict[str, int]: """Check if a prompt exists. @@ -4811,9 +4811,11 @@ def pull_prompt_manifest(self, prompt_identifier: str) -> ls_schemas.PromptManif ) if not use_optimization and commit_hash == "latest": - commit_hash = self._get_latest_commit_hash(f"{owner}/{prompt_name}") - if commit_hash is None: + latest_commit_hash = self._get_latest_commit_hash(f"{owner}/{prompt_name}") + if latest_commit_hash is None: raise ValueError("No commits found") + else: + commit_hash = latest_commit_hash response = self.request_with_retries( "GET", f"/commits/{owner}/{prompt_name}/{commit_hash}" From 5dda8950d8c9dd849b5958fe634b2a07a72fb084 Mon Sep 17 00:00:00 2001 From: Maddy Adams Date: Fri, 12 Jul 2024 16:12:37 -0700 Subject: [PATCH 22/61] update update --- python/langsmith/client.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/python/langsmith/client.py b/python/langsmith/client.py index 6c0df3bab..16ab7a805 100644 --- a/python/langsmith/client.py +++ b/python/langsmith/client.py @@ -4736,16 +4736,20 @@ def update_prompt( prompt_identifier: str, *, description: Optional[str] = None, - is_public: Optional[bool] = None, + readme: Optional[str] = None, tags: Optional[List[str]] = None, + is_public: Optional[bool] = None, + is_archived: Optional[bool] = None, ) -> Dict[str, Any]: """Update a prompt's metadata. Args: prompt_identifier (str): The identifier of the prompt to update. description (Optional[str]): New description for the prompt. - is_public (Optional[bool]): New public status for the prompt. + readme (Optional[str]): New readme for the prompt. tags (Optional[List[str]]): New list of tags for the prompt. + is_public (Optional[bool]): New public status for the prompt. + is_archived (Optional[bool]): New archived status for the prompt. Returns: Dict[str, Any]: The updated prompt data as returned by the server. @@ -4754,11 +4758,19 @@ def update_prompt( ValueError: If the prompt_identifier is empty. HTTPError: If the server request fails. """ + if not prompt_identifier: + raise ValueError("The prompt_identifier cannot be empty.") + json: Dict[str, Union[str, bool, List[str]]] = {} + if description is not None: json["description"] = description + if readme is not None: + json["readme"] = readme if is_public is not None: json["is_public"] = is_public + if is_archived is not None: + json["is_archived"] = is_archived if tags is not None: json["tags"] = tags From bf115e1196bbd73efb5321035d628d642186b62d Mon Sep 17 00:00:00 2001 From: Maddy Adams Date: Fri, 12 Jul 2024 16:18:19 -0700 Subject: [PATCH 23/61] allow passing of readme in push --- python/langsmith/client.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/python/langsmith/client.py b/python/langsmith/client.py index 16ab7a805..327d974ac 100644 --- a/python/langsmith/client.py +++ b/python/langsmith/client.py @@ -4871,7 +4871,9 @@ def push_prompt( manifest_json: Any, parent_commit_hash: Optional[str] = "latest", is_public: bool = False, - description: str = "", + description: Optional[str] = "", + readme: Optional[str] = "", + tags: Optional[List[str]] = [], ) -> str: """Push a prompt manifest to the LangSmith API. @@ -4881,7 +4883,12 @@ def push_prompt( parent_commit_hash (Optional[str]): The parent commit hash. Defaults to "latest". is_public (bool): Whether the prompt should be public. Defaults to False. - description (str): A description of the prompt. Defaults to an empty string. + description (Optional[str]): A description of the prompt. + Defaults to an empty string. + readme (Optional[str]): A readme for the prompt. + Defaults to an empty string. + tags (Optional[List[str]]): A list of tags for the prompt. + Defaults to an empty list. Returns: str: The URL of the pushed prompt. @@ -4921,6 +4928,8 @@ def push_prompt( "repo_handle": prompt_name, "is_public": is_public, "description": description, + "readme": readme, + "tags": tags, }, ) From 4ae94e18874cd7f1f2dd4f9b1f5f90ab92ef6c81 Mon Sep 17 00:00:00 2001 From: Maddy Adams Date: Fri, 12 Jul 2024 17:34:41 -0700 Subject: [PATCH 24/61] add more test cases --- python/langsmith/client.py | 235 +++++++++++++++++++-------- python/tests/prompts/test_prompts.py | 181 +++++++++++++++++++-- 2 files changed, 334 insertions(+), 82 deletions(-) diff --git a/python/langsmith/client.py b/python/langsmith/client.py index 327d974ac..b6fa40b54 100644 --- a/python/langsmith/client.py +++ b/python/langsmith/client.py @@ -4560,7 +4560,7 @@ def _evaluate_strings( **kwargs, ) - def get_settings(self) -> dict: + def _get_settings(self) -> dict: """Get the settings for the current tenant. Returns: @@ -4569,7 +4569,7 @@ def get_settings(self) -> dict: response = self.request_with_retries("GET", "/settings") return response.json() - def current_tenant_is_owner(self, owner: str) -> bool: + def _current_tenant_is_owner(self, owner: str) -> bool: """Check if the current workspace has the same handle as owner. Args: @@ -4578,24 +4578,9 @@ def current_tenant_is_owner(self, owner: str) -> bool: Returns: bool: True if the current tenant is the owner, False otherwise. """ - settings = self.get_settings() + settings = self._get_settings() return owner == "-" or settings["tenant_handle"] == owner - def prompt_exists(self, prompt_identifier: str) -> bool: - """Check if a prompt exists. - - Args: - prompt_identifier (str): The identifier of the prompt. - - Returns: - bool: True if the prompt exists, False otherwise. - """ - try: - self.get_prompt(prompt_identifier) - return True - except requests.exceptions.HTTPError as e: - return e.response.status_code != 404 - def _get_latest_commit_hash( self, prompt_owner_and_name: str, limit: int = 1, offset: int = 0 ) -> Optional[str]: @@ -4640,6 +4625,21 @@ def _like_or_unlike_prompt( response.raise_for_status() return response.json() + def _get_prompt_url(self, prompt_identifier: str) -> str: + """Get a URL for a prompt.""" + owner, prompt_name, commit_hash = ls_utils.parse_prompt_identifier( + prompt_identifier + ) + + if self._current_tenant_is_owner(owner): + return f"{self._host_url}/hub/{owner}/{prompt_name}:{commit_hash[:8]}" + + settings = self._get_settings() + return ( + f"{self._host_url}/prompts/{prompt_name}/{commit_hash[:8]}" + f"?organizationId={settings['id']}" + ) + def like_prompt(self, prompt_identifier: str) -> Dict[str, int]: """Check if a prompt exists. @@ -4664,6 +4664,21 @@ def unlike_prompt(self, prompt_identifier: str) -> Dict[str, int]: """ return self._like_or_unlike_prompt(prompt_identifier, like=False) + def prompt_exists(self, prompt_identifier: str) -> bool: + """Check if a prompt exists. + + Args: + prompt_identifier (str): The identifier of the prompt. + + Returns: + bool: True if the prompt exists, False otherwise. + """ + try: + self.get_prompt(prompt_identifier) + return True + except requests.exceptions.HTTPError as e: + return e.response.status_code != 404 + def list_prompts( self, *, @@ -4731,6 +4746,103 @@ def get_prompt(self, prompt_identifier: str) -> Optional[ls_schemas.Prompt]: response.raise_for_status() return None + def create_prompt( + self, + prompt_identifier: str, + *, + description: Optional[str] = None, + readme: Optional[str] = None, + tags: Optional[List[str]] = None, + is_public: bool = False, + ) -> ls_schemas.Prompt: + """Create a new prompt. + + Does not attach prompt manifest, just creates an empty prompt. + + Args: + prompt_name (str): The name of the prompt. + description (Optional[str]): A description of the prompt. + readme (Optional[str]): A readme for the prompt. + tags (Optional[List[str]]): A list of tags for the prompt. + is_public (bool): Whether the prompt should be public. Defaults to False. + + Returns: + ls_schemas.Prompt: The created prompt object. + + Raises: + ValueError: If the current tenant is not the owner. + HTTPError: If the server request fails. + """ + settings = self._get_settings() + if is_public and not settings.get("tenant_handle"): + raise ValueError( + "Cannot create a public prompt without first\n" + "creating a LangChain Hub handle. " + "You can add a handle by creating a public prompt at:\n" + "https://smith.langchain.com/prompts" + ) + + owner, prompt_name, _ = ls_utils.parse_prompt_identifier(prompt_identifier) + if not self._current_tenant_is_owner(owner=owner): + raise ValueError( + f"Cannot create prompt for another tenant.\n" + f"Current tenant: {self._get_settings()['tenant_handle']},\n" + f"Requested tenant: {owner}" + ) + + json: Dict[str, Union[str, bool, List[str]]] = { + "repo_handle": prompt_name, + "description": description or "", + "readme": readme or "", + "tags": tags or [], + "is_public": is_public, + } + + response = self.request_with_retries("POST", "/repos", json=json) + response.raise_for_status() + return ls_schemas.Prompt(**response.json()["repo"]) + + def create_commit( + self, + prompt_identifier: str, + *, + manifest_json: Any, + parent_commit_hash: Optional[str] = "latest", + ) -> str: + """Create a commit for a prompt. + + Args: + prompt_identifier (str): The identifier of the prompt. + manifest_json (Any): The manifest JSON to commit. + parent_commit_hash (Optional[str]): The hash of the parent commit. + Defaults to "latest". + + Returns: + str: The url of the prompt commit. + + Raises: + HTTPError: If the server request fails. + """ + from langchain_core.load.dump import dumps + + manifest_json = dumps(manifest_json) + manifest_dict = json.loads(manifest_json) + + owner, prompt_name, _ = ls_utils.parse_prompt_identifier(prompt_identifier) + prompt_owner_and_name = f"{owner}/{prompt_name}" + + if parent_commit_hash == "latest": + parent_commit_hash = self._get_latest_commit_hash(prompt_owner_and_name) + + request_dict = {"parent_commit": parent_commit_hash, "manifest": manifest_dict} + response = self.request_with_retries( + "POST", f"/commits/{prompt_owner_and_name}", json=request_dict + ) + + commit_hash = response.json()["commit"]["commit_hash"] + + return self._get_prompt_url(f"{prompt_owner_and_name}:{commit_hash}") + def update_prompt( self, prompt_identifier: str, @@ -4758,8 +4870,14 @@ def update_prompt( ValueError: If the prompt_identifier is empty. HTTPError: If the server request fails. """ - if not prompt_identifier: - raise ValueError("The prompt_identifier cannot be empty.") + settings = self._get_settings() + if is_public and not settings.get("tenant_handle"): + raise ValueError( + "Cannot create a public prompt without first\n" + "creating a LangChain Hub handle. " + "You can add a handle by creating a public prompt at:\n" + "https://smith.langchain.com/prompts" + ) json: Dict[str, Union[str, bool, List[str]]] = {} @@ -4794,10 +4912,10 @@ def delete_prompt(self, prompt_identifier: str) -> bool: ValueError: If the current tenant is not the owner of the prompt. """ owner, prompt_name, _ = ls_utils.parse_prompt_identifier(prompt_identifier) - if not self.current_tenant_is_owner(owner): + if not self._current_tenant_is_owner(owner): raise ValueError( f"Cannot delete prompt for another tenant.\n" - f"Current tenant: {self.get_settings()['tenant_handle']},\n" + f"Current tenant: {self._get_settings()['tenant_handle']},\n" f"Requested tenant: {owner}" ) response = self.request_with_retries("DELETE", f"/repos/{owner}/{prompt_name}") @@ -4868,14 +4986,14 @@ def pull_prompt(self, prompt_identifier: str) -> Any: def push_prompt( self, prompt_identifier: str, - manifest_json: Any, + manifest_json: Optional[Any] = None, parent_commit_hash: Optional[str] = "latest", is_public: bool = False, description: Optional[str] = "", readme: Optional[str] = "", tags: Optional[List[str]] = [], ) -> str: - """Push a prompt manifest to the LangSmith API. + """Push a prompt to the LangSmith API. Args: prompt_identifier (str): The identifier of the prompt. @@ -4897,57 +5015,34 @@ def push_prompt( ValueError: If a public prompt is attempted without a tenant handle or if the current tenant is not the owner. """ - from langchain_core.load.dump import dumps - - manifest_json = dumps(manifest_json) - settings = self.get_settings() - - if is_public and not settings.get("tenant_handle"): - raise ValueError( - "Cannot create a public prompt without first\n" - "creating a LangChain Hub handle. " - "You can add a handle by creating a public prompt at:\n" - "https://smith.langchain.com/prompts" - ) - - owner, prompt_name, _ = ls_utils.parse_prompt_identifier(prompt_identifier) - prompt_full_name = f"{owner}/{prompt_name}" - - if not self.current_tenant_is_owner(owner): - raise ValueError( - "Cannot create prompt for another tenant." - f"Current tenant: {settings['tenant_handle'] or 'no handle'}" - f", Requested tenant: {owner}" + # Create or update prompt metadata + if self.prompt_exists(prompt_identifier): + self.update_prompt( + prompt_identifier, + description=description, + readme=readme, + tags=tags, + is_public=is_public, ) - - if not self.prompt_exists(prompt_full_name): - self.request_with_retries( - "POST", - "/repos/", - json={ - "repo_handle": prompt_name, - "is_public": is_public, - "description": description, - "readme": readme, - "tags": tags, - }, + else: + self.create_prompt( + prompt_identifier, + is_public=is_public, + description=description, + readme=readme, + tags=tags, ) - manifest_dict = json.loads(manifest_json) - if parent_commit_hash == "latest": - parent_commit_hash = self._get_latest_commit_hash(prompt_full_name) + if manifest_json is None: + return self._get_prompt_url(prompt_identifier=prompt_identifier) - request_dict = {"parent_commit": parent_commit_hash, "manifest": manifest_dict} - response = self.request_with_retries( - "POST", f"/commits/{prompt_full_name}", json=request_dict - ) - - commit_hash = response.json()["commit"]["commit_hash"] - short_hash = commit_hash[:8] - return ( - f"{self._host_url}/prompts/{prompt_name}/{short_hash}" - f"?organizationId={settings['id']}" + # Create a commit + url = self.create_commit( + prompt_identifier=prompt_identifier, + manifest_json=manifest_json, + parent_commit_hash=parent_commit_hash, ) + return url def _tracing_thread_drain_queue( diff --git a/python/tests/prompts/test_prompts.py b/python/tests/prompts/test_prompts.py index e0d77c30d..c657ff54a 100644 --- a/python/tests/prompts/test_prompts.py +++ b/python/tests/prompts/test_prompts.py @@ -1,10 +1,10 @@ from uuid import uuid4 import pytest -from langchain_core.prompts import ChatPromptTemplate +from langchain_core.prompts import ChatPromptTemplate, PromptTemplate +import langsmith.schemas as ls_schemas from langsmith.client import Client -from langsmith.schemas import ListPromptsResponse, Prompt, PromptManifest @pytest.fixture @@ -27,16 +27,21 @@ def prompt_template_2() -> ChatPromptTemplate: ) +@pytest.fixture +def prompt_template_3() -> PromptTemplate: + return PromptTemplate.from_template("Summarize the following text: {text}") + + def test_current_tenant_is_owner(langsmith_client: Client): - settings = langsmith_client.get_settings() - assert langsmith_client.current_tenant_is_owner(settings["tenant_handle"]) - assert langsmith_client.current_tenant_is_owner("-") - assert not langsmith_client.current_tenant_is_owner("non_existent_owner") + settings = langsmith_client._get_settings() + assert langsmith_client._current_tenant_is_owner(settings["tenant_handle"]) + assert langsmith_client._current_tenant_is_owner("-") + assert not langsmith_client._current_tenant_is_owner("non_existent_owner") def test_list_prompts(langsmith_client: Client): response = langsmith_client.list_prompts(limit=10, offset=0) - assert isinstance(response, ListPromptsResponse) + assert isinstance(response, ls_schemas.ListPromptsResponse) assert len(response.repos) <= 10 @@ -45,7 +50,7 @@ def test_get_prompt(langsmith_client: Client, prompt_template_1: ChatPromptTempl langsmith_client.push_prompt(prompt_name, prompt_template_1) prompt = langsmith_client.get_prompt(prompt_name) - assert isinstance(prompt, Prompt) + assert isinstance(prompt, ls_schemas.Prompt) assert prompt.repo_handle == prompt_name langsmith_client.delete_prompt(prompt_name) @@ -75,7 +80,7 @@ def test_update_prompt(langsmith_client: Client, prompt_template_1: ChatPromptTe assert isinstance(updated_data, dict) updated_prompt = langsmith_client.get_prompt(prompt_name) - assert isinstance(updated_prompt, Prompt) + assert isinstance(updated_prompt, ls_schemas.Prompt) assert updated_prompt.description == "Updated description" assert updated_prompt.is_public assert set(updated_prompt.tags) == set(["test", "update"]) @@ -99,7 +104,7 @@ def test_pull_prompt_manifest( langsmith_client.push_prompt(prompt_name, prompt_template_1) manifest = langsmith_client.pull_prompt_manifest(prompt_name) - assert isinstance(manifest, PromptManifest) + assert isinstance(manifest, ls_schemas.PromptManifest) assert manifest.repo == prompt_name langsmith_client.delete_prompt(prompt_name) @@ -141,12 +146,12 @@ def test_like_unlike_prompt( langsmith_client.like_prompt(prompt_name) prompt = langsmith_client.get_prompt(prompt_name) - assert isinstance(prompt, Prompt) + assert isinstance(prompt, ls_schemas.Prompt) assert prompt.num_likes == 1 langsmith_client.unlike_prompt(prompt_name) prompt = langsmith_client.get_prompt(prompt_name) - assert isinstance(prompt, Prompt) + assert isinstance(prompt, ls_schemas.Prompt) assert prompt.num_likes == 0 langsmith_client.delete_prompt(prompt_name) @@ -163,3 +168,155 @@ def test_get_latest_commit_hash( assert len(commit_hash) > 0 langsmith_client.delete_prompt(prompt_name) + + +def test_create_prompt(langsmith_client: Client): + prompt_name = f"test_create_prompt_{uuid4().hex[:8]}" + created_prompt = langsmith_client.create_prompt( + prompt_name, + description="Test description", + readme="Test readme", + tags=["test", "create"], + is_public=False, + ) + assert isinstance(created_prompt, ls_schemas.Prompt) + assert created_prompt.repo_handle == prompt_name + assert created_prompt.description == "Test description" + assert created_prompt.readme == "Test readme" + assert set(created_prompt.tags) == set(["test", "create"]) + assert not created_prompt.is_public + + langsmith_client.delete_prompt(prompt_name) + + +def test_create_commit( + langsmith_client: Client, + prompt_template_2: ChatPromptTemplate, + prompt_template_3: PromptTemplate, +): + prompt_name = f"test_create_commit_{uuid4().hex[:8]}" + langsmith_client.push_prompt(prompt_name, prompt_template_3) + commit_url = langsmith_client.create_commit( + prompt_name, manifest_json=prompt_template_2 + ) + assert isinstance(commit_url, str) + assert prompt_name in commit_url + + prompt = langsmith_client.get_prompt(prompt_name) + assert prompt.num_commits == 2 + + langsmith_client.delete_prompt(prompt_name) + + +def test_push_prompt_new(langsmith_client: Client, prompt_template_3: PromptTemplate): + prompt_name = f"test_push_new_{uuid4().hex[:8]}" + url = langsmith_client.push_prompt( + prompt_name, + prompt_template_3, + is_public=True, + description="New prompt", + tags=["new", "test"], + ) + + assert isinstance(url, str) + assert prompt_name in url + + prompt = langsmith_client.get_prompt(prompt_name) + assert prompt.is_public + assert prompt.description == "New prompt" + assert "new" in prompt.tags + assert "test" in prompt.tags + + langsmith_client.delete_prompt(prompt_name) + + +def test_push_prompt_update( + langsmith_client: Client, + prompt_template_1: ChatPromptTemplate, + prompt_template_3: PromptTemplate, +): + prompt_name = f"test_push_update_{uuid4().hex[:8]}" + langsmith_client.push_prompt(prompt_name, prompt_template_1) + + updated_url = langsmith_client.push_prompt( + prompt_name, + prompt_template_3, + description="Updated prompt", + tags=["updated", "test"], + ) + + assert isinstance(updated_url, str) + assert prompt_name in updated_url + + updated_prompt = langsmith_client.get_prompt(prompt_name) + assert updated_prompt.description == "Updated prompt" + assert "updated" in updated_prompt.tags + assert "test" in updated_prompt.tags + + langsmith_client.delete_prompt(prompt_name) + + +@pytest.mark.parametrize("is_public,expected_count", [(True, 1), (False, 1)]) +def test_list_prompts_filter( + langsmith_client: Client, + prompt_template_1: ChatPromptTemplate, + is_public: bool, + expected_count: int, +): + prompt_name = f"test_list_filter_{uuid4().hex[:8]}" + langsmith_client.push_prompt(prompt_name, prompt_template_1, is_public=is_public) + + response = langsmith_client.list_prompts(is_public=is_public, query=prompt_name) + + assert response.total == expected_count + if expected_count > 0: + assert response.repos[0].repo_handle == prompt_name + + langsmith_client.delete_prompt(prompt_name) + + +def test_update_prompt_archive( + langsmith_client: Client, prompt_template_1: ChatPromptTemplate +): + prompt_name = f"test_archive_{uuid4().hex[:8]}" + langsmith_client.push_prompt(prompt_name, prompt_template_1) + + langsmith_client.update_prompt(prompt_name, is_archived=True) + archived_prompt = langsmith_client.get_prompt(prompt_name) + assert archived_prompt.is_archived + + langsmith_client.update_prompt(prompt_name, is_archived=False) + unarchived_prompt = langsmith_client.get_prompt(prompt_name) + assert not unarchived_prompt.is_archived + + langsmith_client.delete_prompt(prompt_name) + + +@pytest.mark.parametrize( + "sort_field,sort_direction", + [ + (ls_schemas.PromptSortField.updated_at, "desc"), + ], +) +def test_list_prompts_sorting( + langsmith_client: Client, + prompt_template_1: ChatPromptTemplate, + sort_field: ls_schemas.PromptSortField, + sort_direction: str, +): + prompt_names = [f"test_sort_{i}_{uuid4().hex[:8]}" for i in range(3)] + for name in prompt_names: + langsmith_client.push_prompt(name, prompt_template_1) + + response = langsmith_client.list_prompts( + sort_field=sort_field, sort_direction=sort_direction, limit=10 + ) + + assert len(response.repos) >= 3 + sorted_names = [ + repo.repo_handle for repo in response.repos if repo.repo_handle in prompt_names + ] + assert sorted_names == sorted(sorted_names, reverse=(sort_direction == "desc")) + + for name in prompt_names: + langsmith_client.delete_prompt(name) From 6ec38be2d5117c52d682b27facf409b1ceb35e4f Mon Sep 17 00:00:00 2001 From: Maddy Adams Date: Fri, 12 Jul 2024 17:41:41 -0700 Subject: [PATCH 25/61] fix tests --- python/tests/prompts/test_prompts.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/python/tests/prompts/test_prompts.py b/python/tests/prompts/test_prompts.py index c657ff54a..e13bce680 100644 --- a/python/tests/prompts/test_prompts.py +++ b/python/tests/prompts/test_prompts.py @@ -1,3 +1,4 @@ +from typing import Literal from uuid import uuid4 import pytest @@ -203,6 +204,7 @@ def test_create_commit( assert prompt_name in commit_url prompt = langsmith_client.get_prompt(prompt_name) + assert isinstance(prompt, ls_schemas.Prompt) assert prompt.num_commits == 2 langsmith_client.delete_prompt(prompt_name) @@ -222,6 +224,7 @@ def test_push_prompt_new(langsmith_client: Client, prompt_template_3: PromptTemp assert prompt_name in url prompt = langsmith_client.get_prompt(prompt_name) + assert isinstance(prompt, ls_schemas.Prompt) assert prompt.is_public assert prompt.description == "New prompt" assert "new" in prompt.tags @@ -249,6 +252,7 @@ def test_push_prompt_update( assert prompt_name in updated_url updated_prompt = langsmith_client.get_prompt(prompt_name) + assert isinstance(updated_prompt, ls_schemas.Prompt) assert updated_prompt.description == "Updated prompt" assert "updated" in updated_prompt.tags assert "test" in updated_prompt.tags @@ -283,19 +287,21 @@ def test_update_prompt_archive( langsmith_client.update_prompt(prompt_name, is_archived=True) archived_prompt = langsmith_client.get_prompt(prompt_name) + assert isinstance(archived_prompt, ls_schemas.Prompt) assert archived_prompt.is_archived langsmith_client.update_prompt(prompt_name, is_archived=False) unarchived_prompt = langsmith_client.get_prompt(prompt_name) + assert isinstance(unarchived_prompt, ls_schemas.Prompt) assert not unarchived_prompt.is_archived langsmith_client.delete_prompt(prompt_name) @pytest.mark.parametrize( - "sort_field,sort_direction", + "sort_field, sort_direction", [ - (ls_schemas.PromptSortField.updated_at, "desc"), + (ls_schemas.PromptSortField.updated_at, Literal["desc"]), ], ) def test_list_prompts_sorting( From c330b89a397472c94a6825d653c4890968a9ea99 Mon Sep 17 00:00:00 2001 From: Maddy Adams Date: Sun, 14 Jul 2024 09:11:47 -0700 Subject: [PATCH 26/61] type hint --- python/tests/prompts/test_prompts.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/tests/prompts/test_prompts.py b/python/tests/prompts/test_prompts.py index e13bce680..ac6801767 100644 --- a/python/tests/prompts/test_prompts.py +++ b/python/tests/prompts/test_prompts.py @@ -301,14 +301,14 @@ def test_update_prompt_archive( @pytest.mark.parametrize( "sort_field, sort_direction", [ - (ls_schemas.PromptSortField.updated_at, Literal["desc"]), + (ls_schemas.PromptSortField.updated_at, "desc"), ], ) def test_list_prompts_sorting( langsmith_client: Client, prompt_template_1: ChatPromptTemplate, sort_field: ls_schemas.PromptSortField, - sort_direction: str, + sort_direction: Literal["asc", "desc"], ): prompt_names = [f"test_sort_{i}_{uuid4().hex[:8]}" for i in range(3)] for name in prompt_names: From 03ccfe01e4b97ccc991c6c9c95f25cc2258f70b7 Mon Sep 17 00:00:00 2001 From: Maddy Adams Date: Sun, 14 Jul 2024 12:02:29 -0700 Subject: [PATCH 27/61] push prompt --- python/langsmith/client.py | 113 ++++++++++++++++++++++-------------- python/langsmith/schemas.py | 2 +- 2 files changed, 70 insertions(+), 45 deletions(-) diff --git a/python/langsmith/client.py b/python/langsmith/client.py index b6fa40b54..1c32f7661 100644 --- a/python/langsmith/client.py +++ b/python/langsmith/client.py @@ -4626,12 +4626,20 @@ def _like_or_unlike_prompt( return response.json() def _get_prompt_url(self, prompt_identifier: str) -> str: - """Get a URL for a prompt.""" + """Get a URL for a prompt. + + Args: + prompt_identifier (str): The identifier of the prompt. + + Returns: + str: The URL for the prompt. + + """ owner, prompt_name, commit_hash = ls_utils.parse_prompt_identifier( prompt_identifier ) - if self._current_tenant_is_owner(owner): + if not self._current_tenant_is_owner(owner): return f"{self._host_url}/hub/{owner}/{prompt_name}:{commit_hash[:8]}" settings = self._get_settings() @@ -4640,20 +4648,23 @@ def _get_prompt_url(self, prompt_identifier: str) -> str: f"?organizationId={settings['id']}" ) - def like_prompt(self, prompt_identifier: str) -> Dict[str, int]: + def _prompt_exists(self, prompt_identifier: str) -> bool: """Check if a prompt exists. Args: prompt_identifier (str): The identifier of the prompt. Returns: - A dictionary with the key 'likes' and the count of likes as the value. - + bool: True if the prompt exists, False otherwise. """ - return self._like_or_unlike_prompt(prompt_identifier, like=True) + try: + self.get_prompt(prompt_identifier) + return True + except requests.exceptions.HTTPError as e: + return e.response.status_code != 404 - def unlike_prompt(self, prompt_identifier: str) -> Dict[str, int]: - """Unlike a prompt. + def like_prompt(self, prompt_identifier: str) -> Dict[str, int]: + """Check if a prompt exists. Args: prompt_identifier (str): The identifier of the prompt. @@ -4662,22 +4673,19 @@ def unlike_prompt(self, prompt_identifier: str) -> Dict[str, int]: A dictionary with the key 'likes' and the count of likes as the value. """ - return self._like_or_unlike_prompt(prompt_identifier, like=False) + return self._like_or_unlike_prompt(prompt_identifier, like=True) - def prompt_exists(self, prompt_identifier: str) -> bool: - """Check if a prompt exists. + def unlike_prompt(self, prompt_identifier: str) -> Dict[str, int]: + """Unlike a prompt. Args: prompt_identifier (str): The identifier of the prompt. Returns: - bool: True if the prompt exists, False otherwise. + A dictionary with the key 'likes' and the count of likes as the value. + """ - try: - self.get_prompt(prompt_identifier) - return True - except requests.exceptions.HTTPError as e: - return e.response.status_code != 404 + return self._like_or_unlike_prompt(prompt_identifier, like=False) def list_prompts( self, @@ -4757,7 +4765,7 @@ def create_prompt( ) -> ls_schemas.Prompt: """Create a new prompt. - Does not attach prompt manifest, just creates an empty prompt. + Does not attach prompt object, just creates an empty prompt. Args: prompt_name (str): The name of the prompt. @@ -4805,15 +4813,15 @@ def create_prompt( def create_commit( self, prompt_identifier: str, + object: dict, *, - manifest_json: Any, parent_commit_hash: Optional[str] = "latest", ) -> str: - """Create a commit for a prompt. + """Create a commit for an existing prompt. Args: prompt_identifier (str): The identifier of the prompt. - manifest_json (Any): The manifest JSON to commit. + object (dict): The LangChain object to commit. parent_commit_hash (Optional[str]): The hash of the parent commit. Defaults to "latest". @@ -4822,11 +4830,23 @@ def create_commit( Raises: HTTPError: If the server request fails. + ValueError: If the prompt does not exist. """ - from langchain_core.load.dump import dumps + if not self._prompt_exists(prompt_identifier): + raise ls_utils.LangSmithNotFoundError( + "Prompt does not exist, you must create it first." + ) + + try: + from langchain_core.load.dump import dumps + except ImportError: + raise ImportError( + "The client.create_commit function requires the langchain_core" + "package to run.\nInstall with pip install langchain_core" + ) - manifest_json = dumps(manifest_json) - manifest_dict = json.loads(manifest_json) + object = dumps(object) + manifest_dict = json.loads(object) owner, prompt_name, _ = ls_utils.parse_prompt_identifier(prompt_identifier) prompt_owner_and_name = f"{owner}/{prompt_name}" @@ -4855,6 +4875,8 @@ def update_prompt( ) -> Dict[str, Any]: """Update a prompt's metadata. + To update the content of a prompt, use push_prompt or create_commit instead. + Args: prompt_identifier (str): The identifier of the prompt to update. description (Optional[str]): New description for the prompt. @@ -4921,14 +4943,14 @@ def delete_prompt(self, prompt_identifier: str) -> bool: response = self.request_with_retries("DELETE", f"/repos/{owner}/{prompt_name}") return response.status_code == 204 - def pull_prompt_manifest(self, prompt_identifier: str) -> ls_schemas.PromptManifest: - """Pull a prompt manifest from the LangSmith API. + def pull_prompt_object(self, prompt_identifier: str) -> ls_schemas.PromptObject: + """Pull a prompt object from the LangSmith API. Args: prompt_identifier (str): The identifier of the prompt. Returns: - ls_schemas.PromptManifest: The prompt manifest. + ls_schemas.PromptObject: The prompt object. Raises: ValueError: If no commits are found for the prompt. @@ -4950,14 +4972,14 @@ def pull_prompt_manifest(self, prompt_identifier: str) -> ls_schemas.PromptManif response = self.request_with_retries( "GET", f"/commits/{owner}/{prompt_name}/{commit_hash}" ) - return ls_schemas.PromptManifest( + return ls_schemas.PromptObject( **{"owner": owner, "repo": prompt_name, **response.json()} ) def pull_prompt(self, prompt_identifier: str) -> Any: """Pull a prompt and return it as a LangChain PromptTemplate. - This method requires `langchain_core` to convert the prompt manifest. + This method requires `langchain_core`. Args: prompt_identifier (str): The identifier of the prompt. @@ -4968,16 +4990,16 @@ def pull_prompt(self, prompt_identifier: str) -> Any: from langchain_core.load.load import loads from langchain_core.prompts import BasePromptTemplate - prompt_manifest = self.pull_prompt_manifest(prompt_identifier) - prompt = loads(json.dumps(prompt_manifest.manifest)) + prompt_object = self.pull_prompt_object(prompt_identifier) + prompt = loads(json.dumps(prompt_object.manifest)) if isinstance(prompt, BasePromptTemplate): if prompt.metadata is None: prompt.metadata = {} prompt.metadata.update( { - "lc_hub_owner": prompt_manifest.owner, - "lc_hub_repo": prompt_manifest.repo, - "lc_hub_commit_hash": prompt_manifest.commit_hash, + "lc_hub_owner": prompt_object.owner, + "lc_hub_repo": prompt_object.repo, + "lc_hub_commit_hash": prompt_object.commit_hash, } ) @@ -4986,7 +5008,8 @@ def pull_prompt(self, prompt_identifier: str) -> Any: def push_prompt( self, prompt_identifier: str, - manifest_json: Optional[Any] = None, + *, + object: Optional[dict] = None, parent_commit_hash: Optional[str] = "latest", is_public: bool = False, description: Optional[str] = "", @@ -4995,9 +5018,14 @@ def push_prompt( ) -> str: """Push a prompt to the LangSmith API. + If the prompt does not exist, it will be created. + If the prompt exists, it will be updated. + + Can be used to update prompt metadata or prompt content. + Args: prompt_identifier (str): The identifier of the prompt. - manifest_json (Any): The manifest to push. + object (Optional[dict]): The LangChain object to push. parent_commit_hash (Optional[str]): The parent commit hash. Defaults to "latest". is_public (bool): Whether the prompt should be public. Defaults to False. @@ -5009,14 +5037,11 @@ def push_prompt( Defaults to an empty list. Returns: - str: The URL of the pushed prompt. + str: The URL of the prompt. - Raises: - ValueError: If a public prompt is attempted without a tenant handle or - if the current tenant is not the owner. """ # Create or update prompt metadata - if self.prompt_exists(prompt_identifier): + if self._prompt_exists(prompt_identifier): self.update_prompt( prompt_identifier, description=description, @@ -5033,13 +5058,13 @@ def push_prompt( tags=tags, ) - if manifest_json is None: + if object is None: return self._get_prompt_url(prompt_identifier=prompt_identifier) - # Create a commit + # Create a commit with the new manifest url = self.create_commit( prompt_identifier=prompt_identifier, - manifest_json=manifest_json, + object=object, parent_commit_hash=parent_commit_hash, ) return url diff --git a/python/langsmith/schemas.py b/python/langsmith/schemas.py index c37b9d14c..6b9894a7f 100644 --- a/python/langsmith/schemas.py +++ b/python/langsmith/schemas.py @@ -746,7 +746,7 @@ def metadata(self) -> dict[str, Any]: return self.extra["metadata"] -class PromptManifest(BaseModel): +class PromptObject(BaseModel): """Represents a Prompt with a manifest. Attributes: From a99c17ada7d06108b63a858f861803cb55794f11 Mon Sep 17 00:00:00 2001 From: Maddy Adams Date: Sun, 14 Jul 2024 12:31:59 -0700 Subject: [PATCH 28/61] pull prompt --- python/langsmith/client.py | 44 ++++++---- python/tests/prompts/test_prompts.py | 124 +++++++++++++++++---------- 2 files changed, 104 insertions(+), 64 deletions(-) diff --git a/python/langsmith/client.py b/python/langsmith/client.py index 1c32f7661..43d6e3cd7 100644 --- a/python/langsmith/client.py +++ b/python/langsmith/client.py @@ -4581,6 +4581,15 @@ def _current_tenant_is_owner(self, owner: str) -> bool: settings = self._get_settings() return owner == "-" or settings["tenant_handle"] == owner + def _owner_conflict_error( + self, action: str, owner: str + ) -> ls_utils.LangSmithUserError: + return ls_utils.LangSmithUserError( + f"Cannot {action} for another tenant.\n" + f"Current tenant: {self._get_settings()['tenant_handle']},\n" + f"Requested tenant: {owner}" + ) + def _get_latest_commit_hash( self, prompt_owner_and_name: str, limit: int = 1, offset: int = 0 ) -> Optional[str]: @@ -4783,7 +4792,7 @@ def create_prompt( """ settings = self._get_settings() if is_public and not settings.get("tenant_handle"): - raise ValueError( + raise ls_utils.LangSmithUserError( "Cannot create a public prompt without first\n" "creating a LangChain Hub handle. " "You can add a handle by creating a public prompt at:\n" @@ -4792,11 +4801,7 @@ def create_prompt( owner, prompt_name, _ = ls_utils.parse_prompt_identifier(prompt_identifier) if not self._current_tenant_is_owner(owner=owner): - raise ValueError( - f"Cannot create prompt for another tenant.\n" - f"Current tenant: {self._get_settings()['tenant_handle']},\n" - f"Requested tenant: {owner}" - ) + raise self._owner_conflict_error("create a prompt", owner) json: Dict[str, Union[str, bool, List[str]]] = { "repo_handle": prompt_name, @@ -4842,7 +4847,7 @@ def create_commit( except ImportError: raise ImportError( "The client.create_commit function requires the langchain_core" - "package to run.\nInstall with pip install langchain_core" + "package to run.\nInstall with `pip install langchain_core`" ) object = dumps(object) @@ -4935,11 +4940,8 @@ def delete_prompt(self, prompt_identifier: str) -> bool: """ owner, prompt_name, _ = ls_utils.parse_prompt_identifier(prompt_identifier) if not self._current_tenant_is_owner(owner): - raise ValueError( - f"Cannot delete prompt for another tenant.\n" - f"Current tenant: {self._get_settings()['tenant_handle']},\n" - f"Requested tenant: {owner}" - ) + raise self._owner_conflict_error("delete a prompt", owner) + response = self.request_with_retries("DELETE", f"/repos/{owner}/{prompt_name}") return response.status_code == 204 @@ -4987,8 +4989,14 @@ def pull_prompt(self, prompt_identifier: str) -> Any: Returns: Any: The prompt object in the specified format. """ - from langchain_core.load.load import loads - from langchain_core.prompts import BasePromptTemplate + try: + from langchain_core.load.load import loads + from langchain_core.prompts import BasePromptTemplate + except ImportError: + raise ImportError( + "The client.pull_prompt function requires the langchain_core" + "package to run.\nInstall with `pip install langchain_core`" + ) prompt_object = self.pull_prompt_object(prompt_identifier) prompt = loads(json.dumps(prompt_object.manifest)) @@ -5018,11 +5026,11 @@ def push_prompt( ) -> str: """Push a prompt to the LangSmith API. + Can be used to update prompt metadata or prompt content. + If the prompt does not exist, it will be created. If the prompt exists, it will be updated. - Can be used to update prompt metadata or prompt content. - Args: prompt_identifier (str): The identifier of the prompt. object (Optional[dict]): The LangChain object to push. @@ -5063,8 +5071,8 @@ def push_prompt( # Create a commit with the new manifest url = self.create_commit( - prompt_identifier=prompt_identifier, - object=object, + prompt_identifier, + object, parent_commit_hash=parent_commit_hash, ) return url diff --git a/python/tests/prompts/test_prompts.py b/python/tests/prompts/test_prompts.py index ac6801767..e235e3d6e 100644 --- a/python/tests/prompts/test_prompts.py +++ b/python/tests/prompts/test_prompts.py @@ -5,6 +5,7 @@ from langchain_core.prompts import ChatPromptTemplate, PromptTemplate import langsmith.schemas as ls_schemas +import langsmith.utils as ls_utils from langsmith.client import Client @@ -48,7 +49,7 @@ def test_list_prompts(langsmith_client: Client): def test_get_prompt(langsmith_client: Client, prompt_template_1: ChatPromptTemplate): prompt_name = f"test_prompt_{uuid4().hex[:8]}" - langsmith_client.push_prompt(prompt_name, prompt_template_1) + langsmith_client.push_prompt(prompt_name, object=prompt_template_1) prompt = langsmith_client.get_prompt(prompt_name) assert isinstance(prompt, ls_schemas.Prompt) @@ -59,18 +60,18 @@ def test_get_prompt(langsmith_client: Client, prompt_template_1: ChatPromptTempl def test_prompt_exists(langsmith_client: Client, prompt_template_2: ChatPromptTemplate): non_existent_prompt = f"non_existent_{uuid4().hex[:8]}" - assert not langsmith_client.prompt_exists(non_existent_prompt) + assert not langsmith_client._prompt_exists(non_existent_prompt) existent_prompt = f"existent_{uuid4().hex[:8]}" - langsmith_client.push_prompt(existent_prompt, prompt_template_2) - assert langsmith_client.prompt_exists(existent_prompt) + langsmith_client.push_prompt(existent_prompt, object=prompt_template_2) + assert langsmith_client._prompt_exists(existent_prompt) langsmith_client.delete_prompt(existent_prompt) def test_update_prompt(langsmith_client: Client, prompt_template_1: ChatPromptTemplate): prompt_name = f"test_prompt_{uuid4().hex[:8]}" - langsmith_client.push_prompt(prompt_name, prompt_template_1) + langsmith_client.push_prompt(prompt_name, object=prompt_template_1) updated_data = langsmith_client.update_prompt( prompt_name, @@ -91,21 +92,21 @@ def test_update_prompt(langsmith_client: Client, prompt_template_1: ChatPromptTe def test_delete_prompt(langsmith_client: Client, prompt_template_1: ChatPromptTemplate): prompt_name = f"test_prompt_{uuid4().hex[:8]}" - langsmith_client.push_prompt(prompt_name, prompt_template_1) + langsmith_client.push_prompt(prompt_name, object=prompt_template_1) - assert langsmith_client.prompt_exists(prompt_name) + assert langsmith_client._prompt_exists(prompt_name) langsmith_client.delete_prompt(prompt_name) - assert not langsmith_client.prompt_exists(prompt_name) + assert not langsmith_client._prompt_exists(prompt_name) -def test_pull_prompt_manifest( +def test_pull_prompt_object( langsmith_client: Client, prompt_template_1: ChatPromptTemplate ): prompt_name = f"test_prompt_{uuid4().hex[:8]}" - langsmith_client.push_prompt(prompt_name, prompt_template_1) + langsmith_client.push_prompt(prompt_name, object=prompt_template_1) - manifest = langsmith_client.pull_prompt_manifest(prompt_name) - assert isinstance(manifest, ls_schemas.PromptManifest) + manifest = langsmith_client.pull_prompt_object(prompt_name) + assert isinstance(manifest, ls_schemas.PromptObject) assert manifest.repo == prompt_name langsmith_client.delete_prompt(prompt_name) @@ -113,10 +114,41 @@ def test_pull_prompt_manifest( def test_pull_prompt(langsmith_client: Client, prompt_template_1: ChatPromptTemplate): prompt_name = f"test_prompt_{uuid4().hex[:8]}" - langsmith_client.push_prompt(prompt_name, prompt_template_1) + langsmith_client.push_prompt(prompt_name, object=prompt_template_1) + # test pulling with just prompt name pulled_prompt = langsmith_client.pull_prompt(prompt_name) assert isinstance(pulled_prompt, ChatPromptTemplate) + assert pulled_prompt.metadata["lc_hub_repo"] == prompt_name + + # test pulling with private owner (-) and name + pulled_prompt_2 = langsmith_client.pull_prompt(f"-/{prompt_name}") + assert pulled_prompt == pulled_prompt_2 + + # test pulling with tenant handle and name + tenant_handle = langsmith_client._get_settings()["tenant_handle"] + pulled_prompt_3 = langsmith_client.pull_prompt(f"{tenant_handle}/{prompt_name}") + assert ( + pulled_prompt.metadata["lc_hub_commit_hash"] + == pulled_prompt_3.metadata["lc_hub_commit_hash"] + ) + assert pulled_prompt_3.metadata["lc_hub_owner"] == tenant_handle + + # test pulling with handle, name and commit hash + tenant_handle = langsmith_client._get_settings()["tenant_handle"] + pulled_prompt_4 = langsmith_client.pull_prompt( + f"{tenant_handle}/{prompt_name}:latest" + ) + assert pulled_prompt_3 == pulled_prompt_4 + + # test pulling without handle, with commit hash + pulled_prompt_5 = langsmith_client.pull_prompt( + f"{prompt_name}:{pulled_prompt_4.metadata['lc_hub_commit_hash']}" + ) + assert ( + pulled_prompt_4.metadata["lc_hub_commit_hash"] + == pulled_prompt_5.metadata["lc_hub_commit_hash"] + ) langsmith_client.delete_prompt(prompt_name) @@ -126,7 +158,7 @@ def test_push_and_pull_prompt( ): prompt_name = f"test_prompt_{uuid4().hex[:8]}" - push_result = langsmith_client.push_prompt(prompt_name, prompt_template_2) + push_result = langsmith_client.push_prompt(prompt_name, object=prompt_template_2) assert isinstance(push_result, str) pulled_prompt = langsmith_client.pull_prompt(prompt_name) @@ -135,15 +167,17 @@ def test_push_and_pull_prompt( langsmith_client.delete_prompt(prompt_name) # should fail - with pytest.raises(ValueError): - langsmith_client.push_prompt(f"random_handle/{prompt_name}", prompt_template_2) + with pytest.raises(ls_utils.LangSmithUserError): + langsmith_client.push_prompt( + f"random_handle/{prompt_name}", object=prompt_template_2 + ) def test_like_unlike_prompt( langsmith_client: Client, prompt_template_1: ChatPromptTemplate ): prompt_name = f"test_prompt_{uuid4().hex[:8]}" - langsmith_client.push_prompt(prompt_name, prompt_template_1) + langsmith_client.push_prompt(prompt_name, object=prompt_template_1) langsmith_client.like_prompt(prompt_name) prompt = langsmith_client.get_prompt(prompt_name) @@ -162,7 +196,7 @@ def test_get_latest_commit_hash( langsmith_client: Client, prompt_template_1: ChatPromptTemplate ): prompt_name = f"test_prompt_{uuid4().hex[:8]}" - langsmith_client.push_prompt(prompt_name, prompt_template_1) + langsmith_client.push_prompt(prompt_name, object=prompt_template_1) commit_hash = langsmith_client._get_latest_commit_hash(f"-/{prompt_name}") assert isinstance(commit_hash, str) @@ -196,10 +230,19 @@ def test_create_commit( prompt_template_3: PromptTemplate, ): prompt_name = f"test_create_commit_{uuid4().hex[:8]}" - langsmith_client.push_prompt(prompt_name, prompt_template_3) - commit_url = langsmith_client.create_commit( - prompt_name, manifest_json=prompt_template_2 - ) + try: + # this should fail because the prompt does not exist + commit_url = langsmith_client.create_commit( + prompt_name, object=prompt_template_2 + ) + pytest.fail("Expected LangSmithNotFoundError was not raised") + except ls_utils.LangSmithNotFoundError as e: + assert str(e) == "Prompt does not exist, you must create it first." + except Exception as e: + pytest.fail(f"Unexpected exception raised: {e}") + + langsmith_client.push_prompt(prompt_name, object=prompt_template_3) + commit_url = langsmith_client.create_commit(prompt_name, object=prompt_template_2) assert isinstance(commit_url, str) assert prompt_name in commit_url @@ -210,11 +253,11 @@ def test_create_commit( langsmith_client.delete_prompt(prompt_name) -def test_push_prompt_new(langsmith_client: Client, prompt_template_3: PromptTemplate): +def test_push_prompt(langsmith_client: Client, prompt_template_3: PromptTemplate): prompt_name = f"test_push_new_{uuid4().hex[:8]}" url = langsmith_client.push_prompt( prompt_name, - prompt_template_3, + object=prompt_template_3, is_public=True, description="New prompt", tags=["new", "test"], @@ -229,33 +272,20 @@ def test_push_prompt_new(langsmith_client: Client, prompt_template_3: PromptTemp assert prompt.description == "New prompt" assert "new" in prompt.tags assert "test" in prompt.tags + assert prompt.num_commits == 1 - langsmith_client.delete_prompt(prompt_name) - - -def test_push_prompt_update( - langsmith_client: Client, - prompt_template_1: ChatPromptTemplate, - prompt_template_3: PromptTemplate, -): - prompt_name = f"test_push_update_{uuid4().hex[:8]}" - langsmith_client.push_prompt(prompt_name, prompt_template_1) - - updated_url = langsmith_client.push_prompt( + # test updating prompt metadata but not manifest + url = langsmith_client.push_prompt( prompt_name, - prompt_template_3, + is_public=False, description="Updated prompt", - tags=["updated", "test"], ) - assert isinstance(updated_url, str) - assert prompt_name in updated_url - updated_prompt = langsmith_client.get_prompt(prompt_name) assert isinstance(updated_prompt, ls_schemas.Prompt) assert updated_prompt.description == "Updated prompt" - assert "updated" in updated_prompt.tags - assert "test" in updated_prompt.tags + assert not updated_prompt.is_public + assert updated_prompt.num_commits == 1 langsmith_client.delete_prompt(prompt_name) @@ -268,7 +298,9 @@ def test_list_prompts_filter( expected_count: int, ): prompt_name = f"test_list_filter_{uuid4().hex[:8]}" - langsmith_client.push_prompt(prompt_name, prompt_template_1, is_public=is_public) + langsmith_client.push_prompt( + prompt_name, object=prompt_template_1, is_public=is_public + ) response = langsmith_client.list_prompts(is_public=is_public, query=prompt_name) @@ -283,7 +315,7 @@ def test_update_prompt_archive( langsmith_client: Client, prompt_template_1: ChatPromptTemplate ): prompt_name = f"test_archive_{uuid4().hex[:8]}" - langsmith_client.push_prompt(prompt_name, prompt_template_1) + langsmith_client.push_prompt(prompt_name, object=prompt_template_1) langsmith_client.update_prompt(prompt_name, is_archived=True) archived_prompt = langsmith_client.get_prompt(prompt_name) @@ -312,7 +344,7 @@ def test_list_prompts_sorting( ): prompt_names = [f"test_sort_{i}_{uuid4().hex[:8]}" for i in range(3)] for name in prompt_names: - langsmith_client.push_prompt(name, prompt_template_1) + langsmith_client.push_prompt(name, object=prompt_template_1) response = langsmith_client.list_prompts( sort_field=sort_field, sort_direction=sort_direction, limit=10 From 3323e3203fb9cd011cbc5ae9d4098a98da10c693 Mon Sep 17 00:00:00 2001 From: Maddy Adams Date: Sun, 14 Jul 2024 12:35:03 -0700 Subject: [PATCH 29/61] any --- python/langsmith/client.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/langsmith/client.py b/python/langsmith/client.py index 43d6e3cd7..22cb80488 100644 --- a/python/langsmith/client.py +++ b/python/langsmith/client.py @@ -5017,7 +5017,7 @@ def push_prompt( self, prompt_identifier: str, *, - object: Optional[dict] = None, + object: Optional[Any] = None, parent_commit_hash: Optional[str] = "latest", is_public: bool = False, description: Optional[str] = "", @@ -5033,7 +5033,7 @@ def push_prompt( Args: prompt_identifier (str): The identifier of the prompt. - object (Optional[dict]): The LangChain object to push. + object (Optional[Any]): The LangChain object to push. parent_commit_hash (Optional[str]): The parent commit hash. Defaults to "latest". is_public (bool): Whether the prompt should be public. Defaults to False. From 1416037e86c301a247288c71ba1eef22a4ced6ed Mon Sep 17 00:00:00 2001 From: Maddy Adams Date: Sun, 14 Jul 2024 12:40:51 -0700 Subject: [PATCH 30/61] lint --- python/langsmith/client.py | 8 ++++---- python/tests/prompts/test_prompts.py | 7 ++++++- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/python/langsmith/client.py b/python/langsmith/client.py index 22cb80488..1dfdf5f90 100644 --- a/python/langsmith/client.py +++ b/python/langsmith/client.py @@ -4818,7 +4818,7 @@ def create_prompt( def create_commit( self, prompt_identifier: str, - object: dict, + object: Any, *, parent_commit_hash: Optional[str] = "latest", ) -> str: @@ -4826,7 +4826,7 @@ def create_commit( Args: prompt_identifier (str): The identifier of the prompt. - object (dict): The LangChain object to commit. + object (Any): The LangChain object to commit. parent_commit_hash (Optional[str]): The hash of the parent commit. Defaults to "latest". @@ -4850,8 +4850,8 @@ def create_commit( "package to run.\nInstall with `pip install langchain_core`" ) - object = dumps(object) - manifest_dict = json.loads(object) + json_object = dumps(object) + manifest_dict = json.loads(json_object) owner, prompt_name, _ = ls_utils.parse_prompt_identifier(prompt_identifier) prompt_owner_and_name = f"{owner}/{prompt_name}" diff --git a/python/tests/prompts/test_prompts.py b/python/tests/prompts/test_prompts.py index e235e3d6e..5c535c461 100644 --- a/python/tests/prompts/test_prompts.py +++ b/python/tests/prompts/test_prompts.py @@ -119,7 +119,9 @@ def test_pull_prompt(langsmith_client: Client, prompt_template_1: ChatPromptTemp # test pulling with just prompt name pulled_prompt = langsmith_client.pull_prompt(prompt_name) assert isinstance(pulled_prompt, ChatPromptTemplate) - assert pulled_prompt.metadata["lc_hub_repo"] == prompt_name + assert ( + pulled_prompt.metadata and pulled_prompt.metadata["lc_hub_repo"] == prompt_name + ) # test pulling with private owner (-) and name pulled_prompt_2 = langsmith_client.pull_prompt(f"-/{prompt_name}") @@ -128,6 +130,7 @@ def test_pull_prompt(langsmith_client: Client, prompt_template_1: ChatPromptTemp # test pulling with tenant handle and name tenant_handle = langsmith_client._get_settings()["tenant_handle"] pulled_prompt_3 = langsmith_client.pull_prompt(f"{tenant_handle}/{prompt_name}") + assert pulled_prompt.metadata and pulled_prompt_3.metadata assert ( pulled_prompt.metadata["lc_hub_commit_hash"] == pulled_prompt_3.metadata["lc_hub_commit_hash"] @@ -142,9 +145,11 @@ def test_pull_prompt(langsmith_client: Client, prompt_template_1: ChatPromptTemp assert pulled_prompt_3 == pulled_prompt_4 # test pulling without handle, with commit hash + assert pulled_prompt_4.metadata pulled_prompt_5 = langsmith_client.pull_prompt( f"{prompt_name}:{pulled_prompt_4.metadata['lc_hub_commit_hash']}" ) + assert pulled_prompt_5.metadata assert ( pulled_prompt_4.metadata["lc_hub_commit_hash"] == pulled_prompt_5.metadata["lc_hub_commit_hash"] From 2b5808f62782541a6ae7a66db4ce7fc2ee5ba966 Mon Sep 17 00:00:00 2001 From: Maddy Adams Date: Mon, 15 Jul 2024 11:56:02 -0700 Subject: [PATCH 31/61] add prompts to integration tests --- python/tests/{prompts => integration_tests}/test_prompts.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename python/tests/{prompts => integration_tests}/test_prompts.py (100%) diff --git a/python/tests/prompts/test_prompts.py b/python/tests/integration_tests/test_prompts.py similarity index 100% rename from python/tests/prompts/test_prompts.py rename to python/tests/integration_tests/test_prompts.py From 2ba3ef680d04c7026f43375defa7ebe72146a81d Mon Sep 17 00:00:00 2001 From: Maddy Adams Date: Mon, 15 Jul 2024 13:34:24 -0700 Subject: [PATCH 32/61] change timeout --- python/tests/integration_tests/test_prompts.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/tests/integration_tests/test_prompts.py b/python/tests/integration_tests/test_prompts.py index 5c535c461..0e292748f 100644 --- a/python/tests/integration_tests/test_prompts.py +++ b/python/tests/integration_tests/test_prompts.py @@ -11,7 +11,7 @@ @pytest.fixture def langsmith_client() -> Client: - return Client() + return Client(timeout_ms=[20_000, 90_000]) @pytest.fixture From c459e631c7f9356bb17940dd5573356a62766674 Mon Sep 17 00:00:00 2001 From: Maddy Adams Date: Mon, 15 Jul 2024 13:37:05 -0700 Subject: [PATCH 33/61] make tuple --- python/tests/integration_tests/test_prompts.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/tests/integration_tests/test_prompts.py b/python/tests/integration_tests/test_prompts.py index 0e292748f..a9f914a3e 100644 --- a/python/tests/integration_tests/test_prompts.py +++ b/python/tests/integration_tests/test_prompts.py @@ -1,4 +1,4 @@ -from typing import Literal +from typing import Literal, Tuple from uuid import uuid4 import pytest @@ -11,7 +11,7 @@ @pytest.fixture def langsmith_client() -> Client: - return Client(timeout_ms=[20_000, 90_000]) + return Client(timeout_ms=Tuple[20_000, 90_000]) @pytest.fixture From 176c350bed05a893599d09248c6ac244655bd539 Mon Sep 17 00:00:00 2001 From: Maddy Adams Date: Mon, 15 Jul 2024 13:40:55 -0700 Subject: [PATCH 34/61] tuple --- python/tests/integration_tests/test_prompts.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/tests/integration_tests/test_prompts.py b/python/tests/integration_tests/test_prompts.py index a9f914a3e..7b9ecf16d 100644 --- a/python/tests/integration_tests/test_prompts.py +++ b/python/tests/integration_tests/test_prompts.py @@ -1,4 +1,4 @@ -from typing import Literal, Tuple +from typing import Literal from uuid import uuid4 import pytest @@ -11,7 +11,7 @@ @pytest.fixture def langsmith_client() -> Client: - return Client(timeout_ms=Tuple[20_000, 90_000]) + return Client(timeout_ms=(20_000, 90_000)) @pytest.fixture From 9ec30758259227c85a9ed6d7c20c173156b18948 Mon Sep 17 00:00:00 2001 From: Maddy Adams Date: Mon, 15 Jul 2024 13:50:10 -0700 Subject: [PATCH 35/61] timeout 50k ms --- python/tests/integration_tests/test_prompts.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/tests/integration_tests/test_prompts.py b/python/tests/integration_tests/test_prompts.py index 7b9ecf16d..e9240e904 100644 --- a/python/tests/integration_tests/test_prompts.py +++ b/python/tests/integration_tests/test_prompts.py @@ -11,7 +11,7 @@ @pytest.fixture def langsmith_client() -> Client: - return Client(timeout_ms=(20_000, 90_000)) + return Client(timeout_ms=(50_000, 90_000)) @pytest.fixture From ae2633e74f0ffaa5787777faa3686812e7ec1ae2 Mon Sep 17 00:00:00 2001 From: Maddy Adams Date: Mon, 15 Jul 2024 15:16:38 -0700 Subject: [PATCH 36/61] add pulling prompt with model --- python/Makefile | 3 - python/langsmith/client.py | 40 +++++-- python/langsmith/schemas.py | 1 + .../tests/integration_tests/test_prompts.py | 111 ++++++++++++++++++ 4 files changed, 142 insertions(+), 13 deletions(-) diff --git a/python/Makefile b/python/Makefile index a8bab2a27..d06830bf9 100644 --- a/python/Makefile +++ b/python/Makefile @@ -18,9 +18,6 @@ doctest: evals: poetry run python -m pytest tests/evaluation -prompts: - poetry run python -m pytest tests/prompts - lint: poetry run ruff check . poetry run mypy . diff --git a/python/langsmith/client.py b/python/langsmith/client.py index 1dfdf5f90..15c0d8455 100644 --- a/python/langsmith/client.py +++ b/python/langsmith/client.py @@ -4926,7 +4926,7 @@ def update_prompt( response.raise_for_status() return response.json() - def delete_prompt(self, prompt_identifier: str) -> bool: + def delete_prompt(self, prompt_identifier: str) -> Any: """Delete a prompt. Args: @@ -4943,9 +4943,15 @@ def delete_prompt(self, prompt_identifier: str) -> bool: raise self._owner_conflict_error("delete a prompt", owner) response = self.request_with_retries("DELETE", f"/repos/{owner}/{prompt_name}") - return response.status_code == 204 - def pull_prompt_object(self, prompt_identifier: str) -> ls_schemas.PromptObject: + return response + + def pull_prompt_object( + self, + prompt_identifier: str, + *, + include_model: Optional[bool] = False, + ) -> ls_schemas.PromptObject: """Pull a prompt object from the LangSmith API. Args: @@ -4972,13 +4978,19 @@ def pull_prompt_object(self, prompt_identifier: str) -> ls_schemas.PromptObject: commit_hash = latest_commit_hash response = self.request_with_retries( - "GET", f"/commits/{owner}/{prompt_name}/{commit_hash}" + "GET", + ( + f"/commits/{owner}/{prompt_name}/{commit_hash}" + f"{'?include_model=true' if include_model else ''}" + ), ) return ls_schemas.PromptObject( **{"owner": owner, "repo": prompt_name, **response.json()} ) - def pull_prompt(self, prompt_identifier: str) -> Any: + def pull_prompt( + self, prompt_identifier: str, *, include_model: Optional[bool] = False + ) -> Any: """Pull a prompt and return it as a LangChain PromptTemplate. This method requires `langchain_core`. @@ -4998,12 +5010,20 @@ def pull_prompt(self, prompt_identifier: str) -> Any: "package to run.\nInstall with `pip install langchain_core`" ) - prompt_object = self.pull_prompt_object(prompt_identifier) + prompt_object = self.pull_prompt_object( + prompt_identifier, include_model=include_model + ) prompt = loads(json.dumps(prompt_object.manifest)) - if isinstance(prompt, BasePromptTemplate): - if prompt.metadata is None: - prompt.metadata = {} - prompt.metadata.update( + + if isinstance(prompt, BasePromptTemplate) or isinstance( + prompt.first, BasePromptTemplate + ): + prompt_template = ( + prompt if isinstance(prompt, BasePromptTemplate) else prompt.first + ) + if prompt_template.metadata is None: + prompt_template.metadata = {} + prompt_template.metadata.update( { "lc_hub_owner": prompt_object.owner, "lc_hub_repo": prompt_object.repo, diff --git a/python/langsmith/schemas.py b/python/langsmith/schemas.py index 6b9894a7f..8970f114d 100644 --- a/python/langsmith/schemas.py +++ b/python/langsmith/schemas.py @@ -750,6 +750,7 @@ class PromptObject(BaseModel): """Represents a Prompt with a manifest. Attributes: + owner (str): The handle of the owner of the prompt. repo (str): The name of the prompt. commit_hash (str): The commit hash of the prompt. manifest (Dict[str, Any]): The manifest of the prompt. diff --git a/python/tests/integration_tests/test_prompts.py b/python/tests/integration_tests/test_prompts.py index e9240e904..92001dc06 100644 --- a/python/tests/integration_tests/test_prompts.py +++ b/python/tests/integration_tests/test_prompts.py @@ -3,6 +3,7 @@ import pytest from langchain_core.prompts import ChatPromptTemplate, PromptTemplate +from langchain_core.runnables.base import RunnableSequence import langsmith.schemas as ls_schemas import langsmith.utils as ls_utils @@ -34,6 +35,101 @@ def prompt_template_3() -> PromptTemplate: return PromptTemplate.from_template("Summarize the following text: {text}") +@pytest.fixture +def prompt_with_model() -> dict: + return { + "id": ["langsmith", "playground", "PromptPlayground"], + "lc": 1, + "type": "constructor", + "kwargs": { + "last": { + "id": ["langchain", "schema", "runnable", "RunnableBinding"], + "lc": 1, + "type": "constructor", + "kwargs": { + "bound": { + "id": ["langchain", "chat_models", "openai", "ChatOpenAI"], + "lc": 1, + "type": "constructor", + "kwargs": { + "openai_api_key": { + "id": ["OPENAI_API_KEY"], + "lc": 1, + "type": "secret", + } + }, + }, + "kwargs": {}, + }, + }, + "first": { + "id": ["langchain", "prompts", "chat", "ChatPromptTemplate"], + "lc": 1, + "type": "constructor", + "kwargs": { + "messages": [ + { + "id": [ + "langchain", + "prompts", + "chat", + "SystemMessagePromptTemplate", + ], + "lc": 1, + "type": "constructor", + "kwargs": { + "prompt": { + "id": [ + "langchain", + "prompts", + "prompt", + "PromptTemplate", + ], + "lc": 1, + "type": "constructor", + "kwargs": { + "template": "You are a chatbot.", + "input_variables": [], + "template_format": "f-string", + }, + } + }, + }, + { + "id": [ + "langchain", + "prompts", + "chat", + "HumanMessagePromptTemplate", + ], + "lc": 1, + "type": "constructor", + "kwargs": { + "prompt": { + "id": [ + "langchain", + "prompts", + "prompt", + "PromptTemplate", + ], + "lc": 1, + "type": "constructor", + "kwargs": { + "template": "{question}", + "input_variables": ["question"], + "template_format": "f-string", + }, + } + }, + }, + ], + "input_variables": ["question"], + }, + }, + }, + } + + def test_current_tenant_is_owner(langsmith_client: Client): settings = langsmith_client._get_settings() assert langsmith_client._current_tenant_is_owner(settings["tenant_handle"]) @@ -178,6 +274,21 @@ def test_push_and_pull_prompt( ) +def test_pull_prompt_include_model(langsmith_client: Client, prompt_with_model: dict): + prompt_name = f"test_prompt_with_model_{uuid4().hex[:8]}" + langsmith_client.push_prompt(prompt_name, object=prompt_with_model) + + pulled_prompt = langsmith_client.pull_prompt(prompt_name, include_model=True) + assert isinstance(pulled_prompt, RunnableSequence) + assert ( + pulled_prompt.first + and pulled_prompt.first.metadata + and pulled_prompt.first.metadata["lc_hub_repo"] == prompt_name + ) + + langsmith_client.delete_prompt(prompt_name) + + def test_like_unlike_prompt( langsmith_client: Client, prompt_template_1: ChatPromptTemplate ): From 6802bf6ff85d2de85bf517208f1b3cc79a6dbd24 Mon Sep 17 00:00:00 2001 From: Maddy Adams Date: Mon, 15 Jul 2024 15:18:45 -0700 Subject: [PATCH 37/61] mark as flaky --- .../tests/integration_tests/test_prompts.py | 20 +++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/python/tests/integration_tests/test_prompts.py b/python/tests/integration_tests/test_prompts.py index 92001dc06..abb103465 100644 --- a/python/tests/integration_tests/test_prompts.py +++ b/python/tests/integration_tests/test_prompts.py @@ -130,19 +130,20 @@ def prompt_with_model() -> dict: } +@pytest.mark.skip(reason="This test is flaky") def test_current_tenant_is_owner(langsmith_client: Client): settings = langsmith_client._get_settings() assert langsmith_client._current_tenant_is_owner(settings["tenant_handle"]) assert langsmith_client._current_tenant_is_owner("-") assert not langsmith_client._current_tenant_is_owner("non_existent_owner") - +@pytest.mark.skip(reason="This test is flaky") def test_list_prompts(langsmith_client: Client): response = langsmith_client.list_prompts(limit=10, offset=0) assert isinstance(response, ls_schemas.ListPromptsResponse) assert len(response.repos) <= 10 - +@pytest.mark.skip(reason="This test is flaky") def test_get_prompt(langsmith_client: Client, prompt_template_1: ChatPromptTemplate): prompt_name = f"test_prompt_{uuid4().hex[:8]}" langsmith_client.push_prompt(prompt_name, object=prompt_template_1) @@ -154,6 +155,7 @@ def test_get_prompt(langsmith_client: Client, prompt_template_1: ChatPromptTempl langsmith_client.delete_prompt(prompt_name) +@pytest.mark.skip(reason="This test is flaky") def test_prompt_exists(langsmith_client: Client, prompt_template_2: ChatPromptTemplate): non_existent_prompt = f"non_existent_{uuid4().hex[:8]}" assert not langsmith_client._prompt_exists(non_existent_prompt) @@ -165,6 +167,7 @@ def test_prompt_exists(langsmith_client: Client, prompt_template_2: ChatPromptTe langsmith_client.delete_prompt(existent_prompt) +@pytest.mark.skip(reason="This test is flaky") def test_update_prompt(langsmith_client: Client, prompt_template_1: ChatPromptTemplate): prompt_name = f"test_prompt_{uuid4().hex[:8]}" langsmith_client.push_prompt(prompt_name, object=prompt_template_1) @@ -186,6 +189,7 @@ def test_update_prompt(langsmith_client: Client, prompt_template_1: ChatPromptTe langsmith_client.delete_prompt(prompt_name) +@pytest.mark.skip(reason="This test is flaky") def test_delete_prompt(langsmith_client: Client, prompt_template_1: ChatPromptTemplate): prompt_name = f"test_prompt_{uuid4().hex[:8]}" langsmith_client.push_prompt(prompt_name, object=prompt_template_1) @@ -195,6 +199,7 @@ def test_delete_prompt(langsmith_client: Client, prompt_template_1: ChatPromptTe assert not langsmith_client._prompt_exists(prompt_name) +@pytest.mark.skip(reason="This test is flaky") def test_pull_prompt_object( langsmith_client: Client, prompt_template_1: ChatPromptTemplate ): @@ -208,6 +213,7 @@ def test_pull_prompt_object( langsmith_client.delete_prompt(prompt_name) +@pytest.mark.skip(reason="This test is flaky") def test_pull_prompt(langsmith_client: Client, prompt_template_1: ChatPromptTemplate): prompt_name = f"test_prompt_{uuid4().hex[:8]}" langsmith_client.push_prompt(prompt_name, object=prompt_template_1) @@ -254,6 +260,7 @@ def test_pull_prompt(langsmith_client: Client, prompt_template_1: ChatPromptTemp langsmith_client.delete_prompt(prompt_name) +@pytest.mark.skip(reason="This test is flaky") def test_push_and_pull_prompt( langsmith_client: Client, prompt_template_2: ChatPromptTemplate ): @@ -274,6 +281,7 @@ def test_push_and_pull_prompt( ) +@pytest.mark.skip(reason="This test is flaky") def test_pull_prompt_include_model(langsmith_client: Client, prompt_with_model: dict): prompt_name = f"test_prompt_with_model_{uuid4().hex[:8]}" langsmith_client.push_prompt(prompt_name, object=prompt_with_model) @@ -289,6 +297,7 @@ def test_pull_prompt_include_model(langsmith_client: Client, prompt_with_model: langsmith_client.delete_prompt(prompt_name) +@pytest.mark.skip(reason="This test is flaky") def test_like_unlike_prompt( langsmith_client: Client, prompt_template_1: ChatPromptTemplate ): @@ -308,6 +317,7 @@ def test_like_unlike_prompt( langsmith_client.delete_prompt(prompt_name) +@pytest.mark.skip(reason="This test is flaky") def test_get_latest_commit_hash( langsmith_client: Client, prompt_template_1: ChatPromptTemplate ): @@ -321,6 +331,7 @@ def test_get_latest_commit_hash( langsmith_client.delete_prompt(prompt_name) +@pytest.mark.skip(reason="This test is flaky") def test_create_prompt(langsmith_client: Client): prompt_name = f"test_create_prompt_{uuid4().hex[:8]}" created_prompt = langsmith_client.create_prompt( @@ -340,6 +351,7 @@ def test_create_prompt(langsmith_client: Client): langsmith_client.delete_prompt(prompt_name) +@pytest.mark.skip(reason="This test is flaky") def test_create_commit( langsmith_client: Client, prompt_template_2: ChatPromptTemplate, @@ -369,6 +381,7 @@ def test_create_commit( langsmith_client.delete_prompt(prompt_name) +@pytest.mark.skip(reason="This test is flaky") def test_push_prompt(langsmith_client: Client, prompt_template_3: PromptTemplate): prompt_name = f"test_push_new_{uuid4().hex[:8]}" url = langsmith_client.push_prompt( @@ -407,6 +420,7 @@ def test_push_prompt(langsmith_client: Client, prompt_template_3: PromptTemplate @pytest.mark.parametrize("is_public,expected_count", [(True, 1), (False, 1)]) +@pytest.mark.skip(reason="This test is flaky") def test_list_prompts_filter( langsmith_client: Client, prompt_template_1: ChatPromptTemplate, @@ -427,6 +441,7 @@ def test_list_prompts_filter( langsmith_client.delete_prompt(prompt_name) +@pytest.mark.skip(reason="This test is flaky") def test_update_prompt_archive( langsmith_client: Client, prompt_template_1: ChatPromptTemplate ): @@ -452,6 +467,7 @@ def test_update_prompt_archive( (ls_schemas.PromptSortField.updated_at, "desc"), ], ) +@pytest.mark.skip(reason="This test is flaky") def test_list_prompts_sorting( langsmith_client: Client, prompt_template_1: ChatPromptTemplate, From 7c01b74fd7d6ad02526d259f2b09c7d79b715a34 Mon Sep 17 00:00:00 2001 From: Maddy Adams Date: Mon, 15 Jul 2024 15:22:32 -0700 Subject: [PATCH 38/61] ci --- python/tests/integration_tests/test_prompts.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/python/tests/integration_tests/test_prompts.py b/python/tests/integration_tests/test_prompts.py index abb103465..6de593e25 100644 --- a/python/tests/integration_tests/test_prompts.py +++ b/python/tests/integration_tests/test_prompts.py @@ -137,12 +137,14 @@ def test_current_tenant_is_owner(langsmith_client: Client): assert langsmith_client._current_tenant_is_owner("-") assert not langsmith_client._current_tenant_is_owner("non_existent_owner") + @pytest.mark.skip(reason="This test is flaky") def test_list_prompts(langsmith_client: Client): response = langsmith_client.list_prompts(limit=10, offset=0) assert isinstance(response, ls_schemas.ListPromptsResponse) assert len(response.repos) <= 10 + @pytest.mark.skip(reason="This test is flaky") def test_get_prompt(langsmith_client: Client, prompt_template_1: ChatPromptTemplate): prompt_name = f"test_prompt_{uuid4().hex[:8]}" @@ -290,6 +292,7 @@ def test_pull_prompt_include_model(langsmith_client: Client, prompt_with_model: assert isinstance(pulled_prompt, RunnableSequence) assert ( pulled_prompt.first + and "metadata" in pulled_prompt.first and pulled_prompt.first.metadata and pulled_prompt.first.metadata["lc_hub_repo"] == prompt_name ) From 1fe51294232e6b224abe5390d49d6ec96ddef09e Mon Sep 17 00:00:00 2001 From: Maddy Adams Date: Mon, 15 Jul 2024 15:29:49 -0700 Subject: [PATCH 39/61] lint --- python/langsmith/wrappers/_openai.py | 12 +++++------ .../tests/integration_tests/test_prompts.py | 20 +++++++++++-------- 2 files changed, 18 insertions(+), 14 deletions(-) diff --git a/python/langsmith/wrappers/_openai.py b/python/langsmith/wrappers/_openai.py index 5b6798e8d..45aec0932 100644 --- a/python/langsmith/wrappers/_openai.py +++ b/python/langsmith/wrappers/_openai.py @@ -114,13 +114,13 @@ def _reduce_choices(choices: List[Choice]) -> dict: "arguments": "", } if chunk.function.name: - message["tool_calls"][index]["function"][ - "name" - ] += chunk.function.name + message["tool_calls"][index]["function"]["name"] += ( + chunk.function.name + ) if chunk.function.arguments: - message["tool_calls"][index]["function"][ - "arguments" - ] += chunk.function.arguments + message["tool_calls"][index]["function"]["arguments"] += ( + chunk.function.arguments + ) return { "index": choices[0].index, "finish_reason": next( diff --git a/python/tests/integration_tests/test_prompts.py b/python/tests/integration_tests/test_prompts.py index 6de593e25..e47f5b5cb 100644 --- a/python/tests/integration_tests/test_prompts.py +++ b/python/tests/integration_tests/test_prompts.py @@ -2,7 +2,11 @@ from uuid import uuid4 import pytest -from langchain_core.prompts import ChatPromptTemplate, PromptTemplate +from langchain_core.prompts import ( + BasePromptTemplate, + ChatPromptTemplate, + PromptTemplate, +) from langchain_core.runnables.base import RunnableSequence import langsmith.schemas as ls_schemas @@ -283,19 +287,19 @@ def test_push_and_pull_prompt( ) -@pytest.mark.skip(reason="This test is flaky") +# @pytest.mark.skip(reason="This test is flaky") def test_pull_prompt_include_model(langsmith_client: Client, prompt_with_model: dict): prompt_name = f"test_prompt_with_model_{uuid4().hex[:8]}" langsmith_client.push_prompt(prompt_name, object=prompt_with_model) pulled_prompt = langsmith_client.pull_prompt(prompt_name, include_model=True) assert isinstance(pulled_prompt, RunnableSequence) - assert ( - pulled_prompt.first - and "metadata" in pulled_prompt.first - and pulled_prompt.first.metadata - and pulled_prompt.first.metadata["lc_hub_repo"] == prompt_name - ) + if getattr(pulled_prompt, "first", None): + first = getattr(pulled_prompt, "first") + assert isinstance(first, BasePromptTemplate) + assert first.metadata and first.metadata["lc_hub_repo"] == prompt_name + else: + assert False, "pulled_prompt.first should exist, incorrect prompt format" langsmith_client.delete_prompt(prompt_name) From eb0c73c92488bf17f35880b727efe2a8212a9a51 Mon Sep 17 00:00:00 2001 From: Maddy Adams Date: Mon, 15 Jul 2024 15:32:48 -0700 Subject: [PATCH 40/61] mark test flaky --- python/tests/integration_tests/test_prompts.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/tests/integration_tests/test_prompts.py b/python/tests/integration_tests/test_prompts.py index e47f5b5cb..3d4787960 100644 --- a/python/tests/integration_tests/test_prompts.py +++ b/python/tests/integration_tests/test_prompts.py @@ -287,7 +287,7 @@ def test_push_and_pull_prompt( ) -# @pytest.mark.skip(reason="This test is flaky") +@pytest.mark.skip(reason="This test is flaky") def test_pull_prompt_include_model(langsmith_client: Client, prompt_with_model: dict): prompt_name = f"test_prompt_with_model_{uuid4().hex[:8]}" langsmith_client.push_prompt(prompt_name, object=prompt_with_model) From 8c54f776dfedd664a2b67d37c462561998450b20 Mon Sep 17 00:00:00 2001 From: Maddy Adams Date: Mon, 15 Jul 2024 15:38:07 -0700 Subject: [PATCH 41/61] merge --- python/langsmith/client.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/python/langsmith/client.py b/python/langsmith/client.py index f31ee6dc3..9e7f2b468 100644 --- a/python/langsmith/client.py +++ b/python/langsmith/client.py @@ -4705,11 +4705,7 @@ def list_prompts( offset: int = 0, is_public: Optional[bool] = None, is_archived: Optional[bool] = False, -<<<<<<< HEAD - sort_field: ls_schemas.PromptsSortField = ls_schemas.PromptsSortField.updated_at, -======= sort_field: ls_schemas.PromptSortField = ls_schemas.PromptSortField.updated_at, ->>>>>>> eb0c73c92488bf17f35880b727efe2a8212a9a51 sort_direction: Literal["desc", "asc"] = "desc", query: Optional[str] = None, ) -> ls_schemas.ListPromptsResponse: From 8eb31c189da1c731596ca23265f932031f5b0fe7 Mon Sep 17 00:00:00 2001 From: Maddy Adams Date: Mon, 15 Jul 2024 15:41:13 -0700 Subject: [PATCH 42/61] reformat --- python/langsmith/wrappers/_openai.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/python/langsmith/wrappers/_openai.py b/python/langsmith/wrappers/_openai.py index 45aec0932..5b6798e8d 100644 --- a/python/langsmith/wrappers/_openai.py +++ b/python/langsmith/wrappers/_openai.py @@ -114,13 +114,13 @@ def _reduce_choices(choices: List[Choice]) -> dict: "arguments": "", } if chunk.function.name: - message["tool_calls"][index]["function"]["name"] += ( - chunk.function.name - ) + message["tool_calls"][index]["function"][ + "name" + ] += chunk.function.name if chunk.function.arguments: - message["tool_calls"][index]["function"]["arguments"] += ( - chunk.function.arguments - ) + message["tool_calls"][index]["function"][ + "arguments" + ] += chunk.function.arguments return { "index": choices[0].index, "finish_reason": next( From d62fa73a7a8d9451a9868dd08a9f39abe7d2136b Mon Sep 17 00:00:00 2001 From: Maddy Adams Date: Mon, 15 Jul 2024 16:15:42 -0700 Subject: [PATCH 43/61] convert tests --- python/langsmith/client.py | 111 ++++++++++++------------ python/langsmith/wrappers/_openai.py | 12 +-- python/tests/unit_tests/test_prompts.py | 50 +++++++++++ 3 files changed, 112 insertions(+), 61 deletions(-) create mode 100644 python/tests/unit_tests/test_prompts.py diff --git a/python/langsmith/client.py b/python/langsmith/client.py index 9e7f2b468..01a3521d2 100644 --- a/python/langsmith/client.py +++ b/python/langsmith/client.py @@ -5099,61 +5099,6 @@ def push_prompt( ) return url - def convert_to_openai_format( - self, messages: Any, stop: Optional[List[str]] = None, **kwargs: Any - ) -> dict: - """Convert a prompt to OpenAI format. - - Requires the `langchain_openai` package to be installed. - - Args: - messages (Any): The messages to convert. - stop (Optional[List[str]]): Stop sequences for the prompt. - **kwargs: Additional arguments for the conversion. - - Returns: - dict: The prompt in OpenAI format. - """ - from langchain_openai import ChatOpenAI - - openai = ChatOpenAI() - - try: - return openai._get_request_payload(messages, stop=stop, **kwargs) - except Exception as e: - print(e) - return None - - def convert_to_anthropic_format( - self, - messages: Any, - model_name: Optional[str] = "claude-2", - stop: Optional[List[str]] = None, - **kwargs: Any, - ) -> dict: - """Convert a prompt to Anthropic format. - - Requires the `langchain_anthropic` package to be installed. - - Args: - messages (Any): The messages to convert. - model_name (Optional[str]): The model name to use. Defaults to "claude-2". - stop (Optional[List[str]]): Stop sequences for the prompt. - **kwargs: Additional arguments for the conversion. - - Returns: - dict: The prompt in Anthropic format. - """ - from langchain_anthropic import ChatAnthropic - - anthropic = ChatAnthropic(model_name=model_name) - - try: - return anthropic._get_request_payload(messages, stop=stop, **kwargs) - except Exception as e: - print(e) - return None - def _tracing_thread_drain_queue( tracing_queue: Queue, limit: int = 100, block: bool = True @@ -5301,3 +5246,59 @@ def _tracing_sub_thread_func( tracing_queue, limit=size_limit, block=False ): _tracing_thread_handle_batch(client, tracing_queue, next_batch) + + +def convert_to_openai_format( + messages: Any, stop: Optional[List[str]] = None, **kwargs: Any +) -> dict: + """Convert a prompt to OpenAI format. + + Requires the `langchain_openai` package to be installed. + + Args: + messages (Any): The messages to convert. + stop (Optional[List[str]]): Stop sequences for the prompt. + **kwargs: Additional arguments for the conversion. + + Returns: + dict: The prompt in OpenAI format. + """ + from langchain_openai import ChatOpenAI + + openai = ChatOpenAI() + + try: + return openai._get_request_payload(messages, stop=stop, **kwargs) + except Exception as e: + raise ls_utils.LangSmithError(f"Error converting to OpenAI format: {e}") + + +def convert_to_anthropic_format( + messages: Any, + model_name: str = "claude-2", + stop: Optional[List[str]] = None, + **kwargs: Any, +) -> dict: + """Convert a prompt to Anthropic format. + + Requires the `langchain_anthropic` package to be installed. + + Args: + messages (Any): The messages to convert. + model_name (Optional[str]): The model name to use. Defaults to "claude-2". + stop (Optional[List[str]]): Stop sequences for the prompt. + **kwargs: Additional arguments for the conversion. + + Returns: + dict: The prompt in Anthropic format. + """ + from langchain_anthropic import ChatAnthropic + + anthropic = ChatAnthropic( + model_name=model_name, timeout=None, stop=stop, base_url=None, api_key=None + ) + + try: + return anthropic._get_request_payload(messages, stop=stop, **kwargs) + except Exception as e: + raise ls_utils.LangSmithError(f"Error converting to Anthropic format: {e}") diff --git a/python/langsmith/wrappers/_openai.py b/python/langsmith/wrappers/_openai.py index 5b6798e8d..45aec0932 100644 --- a/python/langsmith/wrappers/_openai.py +++ b/python/langsmith/wrappers/_openai.py @@ -114,13 +114,13 @@ def _reduce_choices(choices: List[Choice]) -> dict: "arguments": "", } if chunk.function.name: - message["tool_calls"][index]["function"][ - "name" - ] += chunk.function.name + message["tool_calls"][index]["function"]["name"] += ( + chunk.function.name + ) if chunk.function.arguments: - message["tool_calls"][index]["function"][ - "arguments" - ] += chunk.function.arguments + message["tool_calls"][index]["function"]["arguments"] += ( + chunk.function.arguments + ) return { "index": choices[0].index, "finish_reason": next( diff --git a/python/tests/unit_tests/test_prompts.py b/python/tests/unit_tests/test_prompts.py new file mode 100644 index 000000000..e88b0f67d --- /dev/null +++ b/python/tests/unit_tests/test_prompts.py @@ -0,0 +1,50 @@ +import pytest +from langchain_core.prompts import ChatPromptTemplate + +from langsmith.client import convert_to_anthropic_format, convert_to_openai_format + + +@pytest.fixture +def chat_prompt_template(): + return ChatPromptTemplate.from_messages( + [ + ("system", "You are a chatbot"), + ("user", "{question}"), + ] + ) + + +def test_convert_to_openai_format(chat_prompt_template: ChatPromptTemplate): + invoked = chat_prompt_template.invoke({"question": "What is the meaning of life?"}) + + res = convert_to_openai_format( + invoked, + ) + + assert res == { + "messages": [ + {"content": "You are a chatbot", "role": "system"}, + {"content": "What is the meaning of life?", "role": "user"}, + ], + "model": "gpt-3.5-turbo", + "stream": False, + "n": 1, + "temperature": 0.7, + } + + +def test_convert_to_anthropic_format(chat_prompt_template: ChatPromptTemplate): + invoked = chat_prompt_template.invoke({"question": "What is the meaning of life?"}) + + res = convert_to_anthropic_format( + invoked, + ) + + print("Res: ", res) + + assert res == { + "model": "claude-2", + "max_tokens": 1024, + "messages": [{"role": "user", "content": "What is the meaning of life?"}], + "system": "You are a chatbot", + } From 2032fb0d0cf02fc97313714a80f2ac63d1e78ca6 Mon Sep 17 00:00:00 2001 From: Maddy Adams Date: Mon, 15 Jul 2024 18:05:48 -0700 Subject: [PATCH 44/61] rm skip --- python/tests/integration_tests/test_prompts.py | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/python/tests/integration_tests/test_prompts.py b/python/tests/integration_tests/test_prompts.py index 3d4787960..23599d16d 100644 --- a/python/tests/integration_tests/test_prompts.py +++ b/python/tests/integration_tests/test_prompts.py @@ -134,7 +134,6 @@ def prompt_with_model() -> dict: } -@pytest.mark.skip(reason="This test is flaky") def test_current_tenant_is_owner(langsmith_client: Client): settings = langsmith_client._get_settings() assert langsmith_client._current_tenant_is_owner(settings["tenant_handle"]) @@ -142,14 +141,12 @@ def test_current_tenant_is_owner(langsmith_client: Client): assert not langsmith_client._current_tenant_is_owner("non_existent_owner") -@pytest.mark.skip(reason="This test is flaky") def test_list_prompts(langsmith_client: Client): response = langsmith_client.list_prompts(limit=10, offset=0) assert isinstance(response, ls_schemas.ListPromptsResponse) assert len(response.repos) <= 10 -@pytest.mark.skip(reason="This test is flaky") def test_get_prompt(langsmith_client: Client, prompt_template_1: ChatPromptTemplate): prompt_name = f"test_prompt_{uuid4().hex[:8]}" langsmith_client.push_prompt(prompt_name, object=prompt_template_1) @@ -161,7 +158,6 @@ def test_get_prompt(langsmith_client: Client, prompt_template_1: ChatPromptTempl langsmith_client.delete_prompt(prompt_name) -@pytest.mark.skip(reason="This test is flaky") def test_prompt_exists(langsmith_client: Client, prompt_template_2: ChatPromptTemplate): non_existent_prompt = f"non_existent_{uuid4().hex[:8]}" assert not langsmith_client._prompt_exists(non_existent_prompt) @@ -173,7 +169,6 @@ def test_prompt_exists(langsmith_client: Client, prompt_template_2: ChatPromptTe langsmith_client.delete_prompt(existent_prompt) -@pytest.mark.skip(reason="This test is flaky") def test_update_prompt(langsmith_client: Client, prompt_template_1: ChatPromptTemplate): prompt_name = f"test_prompt_{uuid4().hex[:8]}" langsmith_client.push_prompt(prompt_name, object=prompt_template_1) @@ -195,7 +190,6 @@ def test_update_prompt(langsmith_client: Client, prompt_template_1: ChatPromptTe langsmith_client.delete_prompt(prompt_name) -@pytest.mark.skip(reason="This test is flaky") def test_delete_prompt(langsmith_client: Client, prompt_template_1: ChatPromptTemplate): prompt_name = f"test_prompt_{uuid4().hex[:8]}" langsmith_client.push_prompt(prompt_name, object=prompt_template_1) @@ -205,7 +199,6 @@ def test_delete_prompt(langsmith_client: Client, prompt_template_1: ChatPromptTe assert not langsmith_client._prompt_exists(prompt_name) -@pytest.mark.skip(reason="This test is flaky") def test_pull_prompt_object( langsmith_client: Client, prompt_template_1: ChatPromptTemplate ): @@ -219,7 +212,6 @@ def test_pull_prompt_object( langsmith_client.delete_prompt(prompt_name) -@pytest.mark.skip(reason="This test is flaky") def test_pull_prompt(langsmith_client: Client, prompt_template_1: ChatPromptTemplate): prompt_name = f"test_prompt_{uuid4().hex[:8]}" langsmith_client.push_prompt(prompt_name, object=prompt_template_1) @@ -266,7 +258,6 @@ def test_pull_prompt(langsmith_client: Client, prompt_template_1: ChatPromptTemp langsmith_client.delete_prompt(prompt_name) -@pytest.mark.skip(reason="This test is flaky") def test_push_and_pull_prompt( langsmith_client: Client, prompt_template_2: ChatPromptTemplate ): @@ -287,7 +278,6 @@ def test_push_and_pull_prompt( ) -@pytest.mark.skip(reason="This test is flaky") def test_pull_prompt_include_model(langsmith_client: Client, prompt_with_model: dict): prompt_name = f"test_prompt_with_model_{uuid4().hex[:8]}" langsmith_client.push_prompt(prompt_name, object=prompt_with_model) @@ -304,7 +294,6 @@ def test_pull_prompt_include_model(langsmith_client: Client, prompt_with_model: langsmith_client.delete_prompt(prompt_name) -@pytest.mark.skip(reason="This test is flaky") def test_like_unlike_prompt( langsmith_client: Client, prompt_template_1: ChatPromptTemplate ): @@ -324,7 +313,6 @@ def test_like_unlike_prompt( langsmith_client.delete_prompt(prompt_name) -@pytest.mark.skip(reason="This test is flaky") def test_get_latest_commit_hash( langsmith_client: Client, prompt_template_1: ChatPromptTemplate ): @@ -338,7 +326,6 @@ def test_get_latest_commit_hash( langsmith_client.delete_prompt(prompt_name) -@pytest.mark.skip(reason="This test is flaky") def test_create_prompt(langsmith_client: Client): prompt_name = f"test_create_prompt_{uuid4().hex[:8]}" created_prompt = langsmith_client.create_prompt( @@ -358,7 +345,6 @@ def test_create_prompt(langsmith_client: Client): langsmith_client.delete_prompt(prompt_name) -@pytest.mark.skip(reason="This test is flaky") def test_create_commit( langsmith_client: Client, prompt_template_2: ChatPromptTemplate, @@ -388,7 +374,6 @@ def test_create_commit( langsmith_client.delete_prompt(prompt_name) -@pytest.mark.skip(reason="This test is flaky") def test_push_prompt(langsmith_client: Client, prompt_template_3: PromptTemplate): prompt_name = f"test_push_new_{uuid4().hex[:8]}" url = langsmith_client.push_prompt( @@ -427,7 +412,6 @@ def test_push_prompt(langsmith_client: Client, prompt_template_3: PromptTemplate @pytest.mark.parametrize("is_public,expected_count", [(True, 1), (False, 1)]) -@pytest.mark.skip(reason="This test is flaky") def test_list_prompts_filter( langsmith_client: Client, prompt_template_1: ChatPromptTemplate, @@ -448,7 +432,6 @@ def test_list_prompts_filter( langsmith_client.delete_prompt(prompt_name) -@pytest.mark.skip(reason="This test is flaky") def test_update_prompt_archive( langsmith_client: Client, prompt_template_1: ChatPromptTemplate ): @@ -474,7 +457,6 @@ def test_update_prompt_archive( (ls_schemas.PromptSortField.updated_at, "desc"), ], ) -@pytest.mark.skip(reason="This test is flaky") def test_list_prompts_sorting( langsmith_client: Client, prompt_template_1: ChatPromptTemplate, From 9cccb9024978d9db539256fd00d059d36f170b1e Mon Sep 17 00:00:00 2001 From: Maddy Adams Date: Mon, 15 Jul 2024 20:22:05 -0700 Subject: [PATCH 45/61] test improper manifest formats --- .../tests/integration_tests/test_prompts.py | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/python/tests/integration_tests/test_prompts.py b/python/tests/integration_tests/test_prompts.py index 23599d16d..41449e608 100644 --- a/python/tests/integration_tests/test_prompts.py +++ b/python/tests/integration_tests/test_prompts.py @@ -371,6 +371,27 @@ def test_create_commit( assert isinstance(prompt, ls_schemas.Prompt) assert prompt.num_commits == 2 + # try submitting different types of unaccepted manifests + try: + # this should fail + commit_url = langsmith_client.create_commit(prompt_name, object={"hi": "hello"}) + except ls_utils.LangSmithError as e: + err = str(e) + assert "Manifest must have an id field" in err + assert "400 Client Error" in err + except Exception as e: + pytest.fail(f"Unexpected exception raised: {e}") + + try: + # this should fail + commit_url = langsmith_client.create_commit(prompt_name, object={"id": ["hi"]}) + except ls_utils.LangSmithError as e: + err = str(e) + assert "Manifest type hi is not supported" in err + assert "400 Client Error" in err + except Exception as e: + pytest.fail(f"Unexpected exception raised: {e}") + langsmith_client.delete_prompt(prompt_name) From a4f7a91080a227c47d64376e44efdd8a0e561f91 Mon Sep 17 00:00:00 2001 From: Maddy Adams Date: Mon, 15 Jul 2024 20:36:07 -0700 Subject: [PATCH 46/61] testing the tests --- python/tests/integration_tests/test_prompts.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/python/tests/integration_tests/test_prompts.py b/python/tests/integration_tests/test_prompts.py index 41449e608..ddbb373e1 100644 --- a/python/tests/integration_tests/test_prompts.py +++ b/python/tests/integration_tests/test_prompts.py @@ -140,6 +140,11 @@ def test_current_tenant_is_owner(langsmith_client: Client): assert langsmith_client._current_tenant_is_owner("-") assert not langsmith_client._current_tenant_is_owner("non_existent_owner") +def test_current_tenant_is_owner2(langsmith_client: Client): + settings = langsmith_client._get_settings() + assert langsmith_client._current_tenant_is_owner(settings["tenant_handle"]) + assert langsmith_client._current_tenant_is_owner("-") + assert not langsmith_client._current_tenant_is_owner("non_existent_owner") def test_list_prompts(langsmith_client: Client): response = langsmith_client.list_prompts(limit=10, offset=0) From d6b339131d48070a3ce804cf809c59a9df088514 Mon Sep 17 00:00:00 2001 From: Maddy Adams Date: Mon, 15 Jul 2024 20:48:54 -0700 Subject: [PATCH 47/61] testing the tests --- python/langsmith/client.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/langsmith/client.py b/python/langsmith/client.py index 15c0d8455..30688dc6a 100644 --- a/python/langsmith/client.py +++ b/python/langsmith/client.py @@ -4605,7 +4605,7 @@ def _get_latest_commit_hash( """ response = self.request_with_retries( "GET", - f"/commits/{prompt_owner_and_name}/", + f"/commits/{prompt_owner_and_name}", params={"limit": limit, "offset": offset}, ) commits = response.json()["commits"] @@ -4861,7 +4861,7 @@ def create_commit( request_dict = {"parent_commit": parent_commit_hash, "manifest": manifest_dict} response = self.request_with_retries( - "POST", f"/commits/{prompt_owner_and_name}", json=request_dict + "POST", f"/commits/{prompt_owner_and_name}/", json=request_dict ) commit_hash = response.json()["commit"]["commit_hash"] From 1868a9fa8da9d25cbab0c77d969464d779b385f0 Mon Sep 17 00:00:00 2001 From: Maddy Adams Date: Mon, 15 Jul 2024 21:16:44 -0700 Subject: [PATCH 48/61] testing the tests more --- python/langsmith/client.py | 18 +++++++++--------- python/tests/integration_tests/test_prompts.py | 11 ++++------- 2 files changed, 13 insertions(+), 16 deletions(-) diff --git a/python/langsmith/client.py b/python/langsmith/client.py index 30688dc6a..c0220f6e7 100644 --- a/python/langsmith/client.py +++ b/python/langsmith/client.py @@ -4605,7 +4605,7 @@ def _get_latest_commit_hash( """ response = self.request_with_retries( "GET", - f"/commits/{prompt_owner_and_name}", + f"/commits/{prompt_owner_and_name}/", params={"limit": limit, "offset": offset}, ) commits = response.json()["commits"] @@ -4629,7 +4629,7 @@ def _like_or_unlike_prompt( """ owner, prompt_name, _ = ls_utils.parse_prompt_identifier(prompt_identifier) response = self.request_with_retries( - "POST", f"/likes/{owner}/{prompt_name}", json={"like": like} + "POST", f"/likes/{owner}/{prompt_name}/", json={"like": like} ) response.raise_for_status() return response.json() @@ -4737,7 +4737,7 @@ def list_prompts( "match_prefix": "true" if query else None, } - response = self.request_with_retries("GET", "/repos", params=params) + response = self.request_with_retries("GET", "/repos/", params=params) return ls_schemas.ListPromptsResponse(**response.json()) def get_prompt(self, prompt_identifier: str) -> Optional[ls_schemas.Prompt]: @@ -4756,7 +4756,7 @@ def get_prompt(self, prompt_identifier: str) -> Optional[ls_schemas.Prompt]: """ owner, prompt_name, _ = ls_utils.parse_prompt_identifier(prompt_identifier) response = self.request_with_retries( - "GET", f"/repos/{owner}/{prompt_name}", to_ignore=[ls_utils.LangSmithError] + "GET", f"/repos/{owner}/{prompt_name}/", to_ignore=[ls_utils.LangSmithError] ) if response.status_code == 200: return ls_schemas.Prompt(**response.json()["repo"]) @@ -4811,7 +4811,7 @@ def create_prompt( "is_public": is_public, } - response = self.request_with_retries("POST", "/repos", json=json) + response = self.request_with_retries("POST", "/repos/", json=json) response.raise_for_status() return ls_schemas.Prompt(**response.json()["repo"]) @@ -4861,7 +4861,7 @@ def create_commit( request_dict = {"parent_commit": parent_commit_hash, "manifest": manifest_dict} response = self.request_with_retries( - "POST", f"/commits/{prompt_owner_and_name}/", json=request_dict + "POST", f"/commits/{prompt_owner_and_name}", json=request_dict ) commit_hash = response.json()["commit"]["commit_hash"] @@ -4921,7 +4921,7 @@ def update_prompt( owner, prompt_name, _ = ls_utils.parse_prompt_identifier(prompt_identifier) response = self.request_with_retries( - "PATCH", f"/repos/{owner}/{prompt_name}", json=json + "PATCH", f"/repos/{owner}/{prompt_name}/", json=json ) response.raise_for_status() return response.json() @@ -4942,7 +4942,7 @@ def delete_prompt(self, prompt_identifier: str) -> Any: if not self._current_tenant_is_owner(owner): raise self._owner_conflict_error("delete a prompt", owner) - response = self.request_with_retries("DELETE", f"/repos/{owner}/{prompt_name}") + response = self.request_with_retries("DELETE", f"/repos/{owner}/{prompt_name}/") return response @@ -4980,7 +4980,7 @@ def pull_prompt_object( response = self.request_with_retries( "GET", ( - f"/commits/{owner}/{prompt_name}/{commit_hash}" + f"/commits/{owner}/{prompt_name}/{commit_hash}/" f"{'?include_model=true' if include_model else ''}" ), ) diff --git a/python/tests/integration_tests/test_prompts.py b/python/tests/integration_tests/test_prompts.py index ddbb373e1..5e371e768 100644 --- a/python/tests/integration_tests/test_prompts.py +++ b/python/tests/integration_tests/test_prompts.py @@ -140,11 +140,6 @@ def test_current_tenant_is_owner(langsmith_client: Client): assert langsmith_client._current_tenant_is_owner("-") assert not langsmith_client._current_tenant_is_owner("non_existent_owner") -def test_current_tenant_is_owner2(langsmith_client: Client): - settings = langsmith_client._get_settings() - assert langsmith_client._current_tenant_is_owner(settings["tenant_handle"]) - assert langsmith_client._current_tenant_is_owner("-") - assert not langsmith_client._current_tenant_is_owner("non_existent_owner") def test_list_prompts(langsmith_client: Client): response = langsmith_client.list_prompts(limit=10, offset=0) @@ -154,7 +149,9 @@ def test_list_prompts(langsmith_client: Client): def test_get_prompt(langsmith_client: Client, prompt_template_1: ChatPromptTemplate): prompt_name = f"test_prompt_{uuid4().hex[:8]}" - langsmith_client.push_prompt(prompt_name, object=prompt_template_1) + url = langsmith_client.push_prompt(prompt_name, object=prompt_template_1) + assert isinstance(url, str) + assert langsmith_client._prompt_exists(prompt_name) prompt = langsmith_client.get_prompt(prompt_name) assert isinstance(prompt, ls_schemas.Prompt) @@ -168,7 +165,7 @@ def test_prompt_exists(langsmith_client: Client, prompt_template_2: ChatPromptTe assert not langsmith_client._prompt_exists(non_existent_prompt) existent_prompt = f"existent_{uuid4().hex[:8]}" - langsmith_client.push_prompt(existent_prompt, object=prompt_template_2) + assert langsmith_client.push_prompt(existent_prompt, object=prompt_template_2) assert langsmith_client._prompt_exists(existent_prompt) langsmith_client.delete_prompt(existent_prompt) From bb6ee1dfc8f63e7ad3d9c7b09b11d337c034dbfa Mon Sep 17 00:00:00 2001 From: Maddy Adams Date: Mon, 15 Jul 2024 22:02:06 -0700 Subject: [PATCH 49/61] maybe this --- python/langsmith/client.py | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/python/langsmith/client.py b/python/langsmith/client.py index c0220f6e7..a6ee31056 100644 --- a/python/langsmith/client.py +++ b/python/langsmith/client.py @@ -4666,11 +4666,8 @@ def _prompt_exists(self, prompt_identifier: str) -> bool: Returns: bool: True if the prompt exists, False otherwise. """ - try: - self.get_prompt(prompt_identifier) - return True - except requests.exceptions.HTTPError as e: - return e.response.status_code != 404 + prompt = self.get_prompt(prompt_identifier) + return True if prompt else False def like_prompt(self, prompt_identifier: str) -> Dict[str, int]: """Check if a prompt exists. @@ -4755,13 +4752,13 @@ def get_prompt(self, prompt_identifier: str) -> Optional[ls_schemas.Prompt]: another error occurs. """ owner, prompt_name, _ = ls_utils.parse_prompt_identifier(prompt_identifier) - response = self.request_with_retries( - "GET", f"/repos/{owner}/{prompt_name}/", to_ignore=[ls_utils.LangSmithError] - ) - if response.status_code == 200: + try: + response = self.request_with_retries( + "GET", f"/repos/{owner}/{prompt_name}/" + ) return ls_schemas.Prompt(**response.json()["repo"]) - response.raise_for_status() - return None + except ls_utils.LangSmithNotFoundError: + return None def create_prompt( self, From a9e7276a41a3ab4150065aedccb4e75fcbe3f785 Mon Sep 17 00:00:00 2001 From: Maddy Adams Date: Mon, 15 Jul 2024 22:28:13 -0700 Subject: [PATCH 50/61] thought i did this before --- python/langsmith/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/langsmith/client.py b/python/langsmith/client.py index a6ee31056..ac4b01f72 100644 --- a/python/langsmith/client.py +++ b/python/langsmith/client.py @@ -4754,7 +4754,7 @@ def get_prompt(self, prompt_identifier: str) -> Optional[ls_schemas.Prompt]: owner, prompt_name, _ = ls_utils.parse_prompt_identifier(prompt_identifier) try: response = self.request_with_retries( - "GET", f"/repos/{owner}/{prompt_name}/" + "GET", f"/repos/{owner}/{prompt_name}" ) return ls_schemas.Prompt(**response.json()["repo"]) except ls_utils.LangSmithNotFoundError: From 43e20d225e55830ccf9e33d765f27ce8f9e20f7a Mon Sep 17 00:00:00 2001 From: Maddy Adams Date: Mon, 15 Jul 2024 22:36:09 -0700 Subject: [PATCH 51/61] they should literally all match --- python/langsmith/client.py | 12 +++++------- python/langsmith/utils.py | 4 ++-- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/python/langsmith/client.py b/python/langsmith/client.py index 68280d2fc..42b8c2e37 100644 --- a/python/langsmith/client.py +++ b/python/langsmith/client.py @@ -4733,7 +4733,7 @@ def _like_or_unlike_prompt( """ owner, prompt_name, _ = ls_utils.parse_prompt_identifier(prompt_identifier) response = self.request_with_retries( - "POST", f"/likes/{owner}/{prompt_name}/", json={"like": like} + "POST", f"/likes/{owner}/{prompt_name}", json={"like": like} ) response.raise_for_status() return response.json() @@ -4857,9 +4857,7 @@ def get_prompt(self, prompt_identifier: str) -> Optional[ls_schemas.Prompt]: """ owner, prompt_name, _ = ls_utils.parse_prompt_identifier(prompt_identifier) try: - response = self.request_with_retries( - "GET", f"/repos/{owner}/{prompt_name}" - ) + response = self.request_with_retries("GET", f"/repos/{owner}/{prompt_name}") return ls_schemas.Prompt(**response.json()["repo"]) except ls_utils.LangSmithNotFoundError: return None @@ -5022,7 +5020,7 @@ def update_prompt( owner, prompt_name, _ = ls_utils.parse_prompt_identifier(prompt_identifier) response = self.request_with_retries( - "PATCH", f"/repos/{owner}/{prompt_name}/", json=json + "PATCH", f"/repos/{owner}/{prompt_name}", json=json ) response.raise_for_status() return response.json() @@ -5043,7 +5041,7 @@ def delete_prompt(self, prompt_identifier: str) -> Any: if not self._current_tenant_is_owner(owner): raise self._owner_conflict_error("delete a prompt", owner) - response = self.request_with_retries("DELETE", f"/repos/{owner}/{prompt_name}/") + response = self.request_with_retries("DELETE", f"/repos/{owner}/{prompt_name}") return response @@ -5081,7 +5079,7 @@ def pull_prompt_object( response = self.request_with_retries( "GET", ( - f"/commits/{owner}/{prompt_name}/{commit_hash}/" + f"/commits/{owner}/{prompt_name}/{commit_hash}" f"{'?include_model=true' if include_model else ''}" ), ) diff --git a/python/langsmith/utils.py b/python/langsmith/utils.py index 39b368e7a..9fecdc04f 100644 --- a/python/langsmith/utils.py +++ b/python/langsmith/utils.py @@ -571,7 +571,6 @@ def deepish_copy(val: T) -> T: return _middle_copy(val, memo) - def is_version_greater_or_equal(current_version, target_version): """Check if the current version is greater or equal to the target version.""" from packaging import version @@ -615,6 +614,7 @@ def parse_prompt_identifier(identifier: str) -> Tuple[str, str, str]: raise ValueError(f"Invalid identifier format: {identifier}") return "-", owner_name, commit + P = ParamSpec("P") @@ -686,4 +686,4 @@ def _wrapped_fn(*args: Any) -> T: *iterables, timeout=timeout, chunksize=chunksize, - ) \ No newline at end of file + ) From 8e85f9b6433b737bf063dceee8b5c9cc7ddd47c3 Mon Sep 17 00:00:00 2001 From: Maddy Adams Date: Tue, 16 Jul 2024 09:51:23 -0700 Subject: [PATCH 52/61] fix tests --- python/langsmith/client.py | 21 ++++++-- .../tests/integration_tests/test_prompts.py | 50 ++++++++++++++++++- python/tests/unit_tests/test_prompts.py | 50 ------------------- 3 files changed, 66 insertions(+), 55 deletions(-) delete mode 100644 python/tests/unit_tests/test_prompts.py diff --git a/python/langsmith/client.py b/python/langsmith/client.py index ebcb0f88f..6c9b62968 100644 --- a/python/langsmith/client.py +++ b/python/langsmith/client.py @@ -5345,7 +5345,7 @@ def _tracing_sub_thread_func( _tracing_thread_handle_batch(client, tracing_queue, next_batch) -def convert_to_openai_format( +def convert_prompt_to_openai_format( messages: Any, stop: Optional[List[str]] = None, **kwargs: Any ) -> dict: """Convert a prompt to OpenAI format. @@ -5360,7 +5360,13 @@ def convert_to_openai_format( Returns: dict: The prompt in OpenAI format. """ - from langchain_openai import ChatOpenAI + try: + from langchain_openai import ChatOpenAI + except ImportError: + raise ImportError( + "The convert_prompt_to_openai_format function requires the langchain_openai" + "package to run.\nInstall with `pip install langchain_openai`" + ) openai = ChatOpenAI() @@ -5370,7 +5376,7 @@ def convert_to_openai_format( raise ls_utils.LangSmithError(f"Error converting to OpenAI format: {e}") -def convert_to_anthropic_format( +def convert_prompt_to_anthropic_format( messages: Any, model_name: str = "claude-2", stop: Optional[List[str]] = None, @@ -5389,7 +5395,14 @@ def convert_to_anthropic_format( Returns: dict: The prompt in Anthropic format. """ - from langchain_anthropic import ChatAnthropic + try: + from langchain_anthropic import ChatAnthropic + except ImportError: + raise ImportError( + "The convert_prompt_to_anthropic_format function requires the " + "langchain_anthropic package to run.\n" + "Install with `pip install langchain_anthropic`" + ) anthropic = ChatAnthropic( model_name=model_name, timeout=None, stop=stop, base_url=None, api_key=None diff --git a/python/tests/integration_tests/test_prompts.py b/python/tests/integration_tests/test_prompts.py index 5e371e768..ece7f072a 100644 --- a/python/tests/integration_tests/test_prompts.py +++ b/python/tests/integration_tests/test_prompts.py @@ -11,7 +11,11 @@ import langsmith.schemas as ls_schemas import langsmith.utils as ls_utils -from langsmith.client import Client +from langsmith.client import ( + Client, + convert_prompt_to_anthropic_format, + convert_prompt_to_openai_format, +) @pytest.fixture @@ -134,6 +138,16 @@ def prompt_with_model() -> dict: } +@pytest.fixture +def chat_prompt_template(): + return ChatPromptTemplate.from_messages( + [ + ("system", "You are a chatbot"), + ("user", "{question}"), + ] + ) + + def test_current_tenant_is_owner(langsmith_client: Client): settings = langsmith_client._get_settings() assert langsmith_client._current_tenant_is_owner(settings["tenant_handle"]) @@ -502,3 +516,37 @@ def test_list_prompts_sorting( for name in prompt_names: langsmith_client.delete_prompt(name) + + +def test_convert_to_openai_format(chat_prompt_template: ChatPromptTemplate): + invoked = chat_prompt_template.invoke({"question": "What is the meaning of life?"}) + + res = convert_prompt_to_openai_format( + invoked, + ) + + assert res == { + "messages": [ + {"content": "You are a chatbot", "role": "system"}, + {"content": "What is the meaning of life?", "role": "user"}, + ], + "model": "gpt-3.5-turbo", + "stream": False, + "n": 1, + "temperature": 0.7, + } + + +def test_convert_to_anthropic_format(chat_prompt_template: ChatPromptTemplate): + invoked = chat_prompt_template.invoke({"question": "What is the meaning of life?"}) + + res = convert_prompt_to_anthropic_format( + invoked, + ) + + assert res == { + "model": "claude-2", + "max_tokens": 1024, + "messages": [{"role": "user", "content": "What is the meaning of life?"}], + "system": "You are a chatbot", + } diff --git a/python/tests/unit_tests/test_prompts.py b/python/tests/unit_tests/test_prompts.py deleted file mode 100644 index e88b0f67d..000000000 --- a/python/tests/unit_tests/test_prompts.py +++ /dev/null @@ -1,50 +0,0 @@ -import pytest -from langchain_core.prompts import ChatPromptTemplate - -from langsmith.client import convert_to_anthropic_format, convert_to_openai_format - - -@pytest.fixture -def chat_prompt_template(): - return ChatPromptTemplate.from_messages( - [ - ("system", "You are a chatbot"), - ("user", "{question}"), - ] - ) - - -def test_convert_to_openai_format(chat_prompt_template: ChatPromptTemplate): - invoked = chat_prompt_template.invoke({"question": "What is the meaning of life?"}) - - res = convert_to_openai_format( - invoked, - ) - - assert res == { - "messages": [ - {"content": "You are a chatbot", "role": "system"}, - {"content": "What is the meaning of life?", "role": "user"}, - ], - "model": "gpt-3.5-turbo", - "stream": False, - "n": 1, - "temperature": 0.7, - } - - -def test_convert_to_anthropic_format(chat_prompt_template: ChatPromptTemplate): - invoked = chat_prompt_template.invoke({"question": "What is the meaning of life?"}) - - res = convert_to_anthropic_format( - invoked, - ) - - print("Res: ", res) - - assert res == { - "model": "claude-2", - "max_tokens": 1024, - "messages": [{"role": "user", "content": "What is the meaning of life?"}], - "system": "You are a chatbot", - } From ff1ff50e4673a2a123f3f9e95eb89124740ab82c Mon Sep 17 00:00:00 2001 From: Maddy Adams Date: Tue, 16 Jul 2024 10:16:21 -0700 Subject: [PATCH 53/61] add dependencies --- .github/workflows/python_test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python_test.yml b/.github/workflows/python_test.yml index 5a45962ae..98020f18e 100644 --- a/.github/workflows/python_test.yml +++ b/.github/workflows/python_test.yml @@ -44,7 +44,7 @@ jobs: - name: Install dependencies run: | poetry install --with dev,lint - poetry run pip install -U langchain langchain-core + poetry run pip install -U langchain langchain-core langchain_anthropic langchain_openai - name: Build ${{ matrix.python-version }} run: poetry build - name: Lint ${{ matrix.python-version }} From 046797dd706f862056ddaa4ed9890ec83f3dffce Mon Sep 17 00:00:00 2001 From: Maddy Adams Date: Tue, 16 Jul 2024 10:18:47 -0700 Subject: [PATCH 54/61] format --- python/langsmith/wrappers/_openai.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/python/langsmith/wrappers/_openai.py b/python/langsmith/wrappers/_openai.py index 45aec0932..5b6798e8d 100644 --- a/python/langsmith/wrappers/_openai.py +++ b/python/langsmith/wrappers/_openai.py @@ -114,13 +114,13 @@ def _reduce_choices(choices: List[Choice]) -> dict: "arguments": "", } if chunk.function.name: - message["tool_calls"][index]["function"]["name"] += ( - chunk.function.name - ) + message["tool_calls"][index]["function"][ + "name" + ] += chunk.function.name if chunk.function.arguments: - message["tool_calls"][index]["function"]["arguments"] += ( - chunk.function.arguments - ) + message["tool_calls"][index]["function"][ + "arguments" + ] += chunk.function.arguments return { "index": choices[0].index, "finish_reason": next( From 77ee8a5da5b4c2242786c5b3bdbcd274f20bde9d Mon Sep 17 00:00:00 2001 From: Maddy Adams Date: Tue, 16 Jul 2024 11:55:47 -0700 Subject: [PATCH 55/61] comments --- python/langsmith/client.py | 35 +++++++++++++------ python/langsmith/schemas.py | 2 +- python/langsmith/utils.py | 2 +- .../tests/integration_tests/test_prompts.py | 4 +-- 4 files changed, 29 insertions(+), 14 deletions(-) diff --git a/python/langsmith/client.py b/python/langsmith/client.py index 42b8c2e37..c19178e20 100644 --- a/python/langsmith/client.py +++ b/python/langsmith/client.py @@ -4664,6 +4664,7 @@ def _evaluate_strings( **kwargs, ) + @functools.lru_cache(maxsize=1) def _get_settings(self) -> dict: """Get the settings for the current tenant. @@ -5025,7 +5026,7 @@ def update_prompt( response.raise_for_status() return response.json() - def delete_prompt(self, prompt_identifier: str) -> Any: + def delete_prompt(self, prompt_identifier: str) -> None: """Delete a prompt. Args: @@ -5042,15 +5043,14 @@ def delete_prompt(self, prompt_identifier: str) -> Any: raise self._owner_conflict_error("delete a prompt", owner) response = self.request_with_retries("DELETE", f"/repos/{owner}/{prompt_name}") + response.raise_for_status() - return response - - def pull_prompt_object( + def pull_prompt_commit( self, prompt_identifier: str, *, include_model: Optional[bool] = False, - ) -> ls_schemas.PromptObject: + ) -> ls_schemas.PromptCommit: """Pull a prompt object from the LangSmith API. Args: @@ -5083,7 +5083,7 @@ def pull_prompt_object( f"{'?include_model=true' if include_model else ''}" ), ) - return ls_schemas.PromptObject( + return ls_schemas.PromptCommit( **{"owner": owner, "repo": prompt_name, **response.json()} ) @@ -5103,23 +5103,38 @@ def pull_prompt( try: from langchain_core.load.load import loads from langchain_core.prompts import BasePromptTemplate + from langchain_core.runnables.base import RunnableSequence except ImportError: raise ImportError( "The client.pull_prompt function requires the langchain_core" "package to run.\nInstall with `pip install langchain_core`" ) - prompt_object = self.pull_prompt_object( + prompt_object = self.pull_prompt_commit( prompt_identifier, include_model=include_model ) prompt = loads(json.dumps(prompt_object.manifest)) - if isinstance(prompt, BasePromptTemplate) or isinstance( - prompt.first, BasePromptTemplate + if ( + isinstance(prompt, BasePromptTemplate) + or isinstance(prompt, RunnableSequence) + and isinstance(prompt.first, BasePromptTemplate) ): prompt_template = ( - prompt if isinstance(prompt, BasePromptTemplate) else prompt.first + prompt + if isinstance(prompt, BasePromptTemplate) + else ( + prompt.first + if isinstance(prompt, RunnableSequence) + and isinstance(prompt.first, BasePromptTemplate) + else None + ) ) + if prompt_template is None: + raise ls_utils.LangSmithError( + "Prompt object is not a valid prompt template." + ) + if prompt_template.metadata is None: prompt_template.metadata = {} prompt_template.metadata.update( diff --git a/python/langsmith/schemas.py b/python/langsmith/schemas.py index 8970f114d..f7eb3d955 100644 --- a/python/langsmith/schemas.py +++ b/python/langsmith/schemas.py @@ -746,7 +746,7 @@ def metadata(self) -> dict[str, Any]: return self.extra["metadata"] -class PromptObject(BaseModel): +class PromptCommit(BaseModel): """Represents a Prompt with a manifest. Attributes: diff --git a/python/langsmith/utils.py b/python/langsmith/utils.py index 9fecdc04f..2456af92a 100644 --- a/python/langsmith/utils.py +++ b/python/langsmith/utils.py @@ -571,7 +571,7 @@ def deepish_copy(val: T) -> T: return _middle_copy(val, memo) -def is_version_greater_or_equal(current_version, target_version): +def is_version_greater_or_equal(current_version: str, target_version: str) -> bool: """Check if the current version is greater or equal to the target version.""" from packaging import version diff --git a/python/tests/integration_tests/test_prompts.py b/python/tests/integration_tests/test_prompts.py index 5e371e768..9d3db4a2b 100644 --- a/python/tests/integration_tests/test_prompts.py +++ b/python/tests/integration_tests/test_prompts.py @@ -207,8 +207,8 @@ def test_pull_prompt_object( prompt_name = f"test_prompt_{uuid4().hex[:8]}" langsmith_client.push_prompt(prompt_name, object=prompt_template_1) - manifest = langsmith_client.pull_prompt_object(prompt_name) - assert isinstance(manifest, ls_schemas.PromptObject) + manifest = langsmith_client.pull_prompt_commit(prompt_name) + assert isinstance(manifest, ls_schemas.PromptCommit) assert manifest.repo == prompt_name langsmith_client.delete_prompt(prompt_name) From a89b514c7cf954e85a5e632fa767fbe4c11da75f Mon Sep 17 00:00:00 2001 From: Maddy Adams Date: Tue, 16 Jul 2024 12:56:58 -0700 Subject: [PATCH 56/61] cache --- python/langsmith/client.py | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/python/langsmith/client.py b/python/langsmith/client.py index c19178e20..b726d926e 100644 --- a/python/langsmith/client.py +++ b/python/langsmith/client.py @@ -472,6 +472,7 @@ class Client: "_hide_outputs", "_info", "_write_api_urls", + "_settings", ] def __init__( @@ -614,6 +615,8 @@ def __init__( else ls_utils.get_env_var("HIDE_OUTPUTS") == "true" ) + self._settings = None + def _repr_html_(self) -> str: """Return an HTML representation of the instance with a link to the URL. @@ -701,6 +704,18 @@ def info(self) -> ls_schemas.LangSmithInfo: self._info = ls_schemas.LangSmithInfo() return self._info + def _get_settings(self) -> dict: + """Get the settings for the current tenant. + + Returns: + dict: The settings for the current tenant. + """ + if self._settings is None: + response = self.request_with_retries("GET", "/settings") + self._settings = response.json() + + return self._settings + def request_with_retries( self, /, @@ -4664,16 +4679,6 @@ def _evaluate_strings( **kwargs, ) - @functools.lru_cache(maxsize=1) - def _get_settings(self) -> dict: - """Get the settings for the current tenant. - - Returns: - dict: The settings for the current tenant. - """ - response = self.request_with_retries("GET", "/settings") - return response.json() - def _current_tenant_is_owner(self, owner: str) -> bool: """Check if the current workspace has the same handle as owner. From cb58952a65f365ffb5d294611398ae71162edf0f Mon Sep 17 00:00:00 2001 From: Maddy Adams Date: Tue, 16 Jul 2024 12:59:48 -0700 Subject: [PATCH 57/61] fix types --- python/langsmith/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/langsmith/client.py b/python/langsmith/client.py index 6dc454fcc..959365132 100644 --- a/python/langsmith/client.py +++ b/python/langsmith/client.py @@ -615,7 +615,7 @@ def __init__( else ls_utils.get_env_var("HIDE_OUTPUTS") == "true" ) - self._settings = None + self._settings = self._get_settings() def _repr_html_(self) -> str: """Return an HTML representation of the instance with a link to the URL. From 667833615ca2c7b2243feafa0018693cfad44ec0 Mon Sep 17 00:00:00 2001 From: Maddy Adams Date: Tue, 16 Jul 2024 13:07:29 -0700 Subject: [PATCH 58/61] fix types --- python/langsmith/client.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/python/langsmith/client.py b/python/langsmith/client.py index 959365132..4011ae287 100644 --- a/python/langsmith/client.py +++ b/python/langsmith/client.py @@ -615,7 +615,7 @@ def __init__( else ls_utils.get_env_var("HIDE_OUTPUTS") == "true" ) - self._settings = self._get_settings() + self._settings: Union[dict, None] = None def _repr_html_(self) -> str: """Return an HTML representation of the instance with a link to the URL. @@ -712,7 +712,8 @@ def _get_settings(self) -> dict: """ if self._settings is None: response = self.request_with_retries("GET", "/settings") - self._settings = response.json() + settings: dict = response.json() + self._settings = settings return self._settings From 2ade88b54a88a6280b520359575371c6f2b67cce Mon Sep 17 00:00:00 2001 From: Maddy Adams Date: Tue, 16 Jul 2024 14:32:18 -0700 Subject: [PATCH 59/61] fixes --- python/langsmith/client.py | 83 +++++++++++-------- python/langsmith/schemas.py | 12 +++ .../tests/integration_tests/test_prompts.py | 8 +- 3 files changed, 63 insertions(+), 40 deletions(-) diff --git a/python/langsmith/client.py b/python/langsmith/client.py index 4011ae287..d46bd75c5 100644 --- a/python/langsmith/client.py +++ b/python/langsmith/client.py @@ -615,7 +615,7 @@ def __init__( else ls_utils.get_env_var("HIDE_OUTPUTS") == "true" ) - self._settings: Union[dict, None] = None + self._settings: Union[ls_schemas.LangSmithSettings, None] = None def _repr_html_(self) -> str: """Return an HTML representation of the instance with a link to the URL. @@ -704,7 +704,7 @@ def info(self) -> ls_schemas.LangSmithInfo: self._info = ls_schemas.LangSmithInfo() return self._info - def _get_settings(self) -> dict: + def _get_settings(self) -> ls_schemas.LangSmithSettings: """Get the settings for the current tenant. Returns: @@ -712,8 +712,8 @@ def _get_settings(self) -> dict: """ if self._settings is None: response = self.request_with_retries("GET", "/settings") - settings: dict = response.json() - self._settings = settings + ls_utils.raise_for_status_with_text(response) + self._settings = ls_schemas.LangSmithSettings(**response.json()) return self._settings @@ -4690,14 +4690,14 @@ def _current_tenant_is_owner(self, owner: str) -> bool: bool: True if the current tenant is the owner, False otherwise. """ settings = self._get_settings() - return owner == "-" or settings["tenant_handle"] == owner + return owner == "-" or settings.tenant_handle == owner def _owner_conflict_error( self, action: str, owner: str ) -> ls_utils.LangSmithUserError: return ls_utils.LangSmithUserError( f"Cannot {action} for another tenant.\n" - f"Current tenant: {self._get_settings()['tenant_handle']},\n" + f"Current tenant: {self._get_settings().tenant_handle},\n" f"Requested tenant: {owner}" ) @@ -4765,7 +4765,7 @@ def _get_prompt_url(self, prompt_identifier: str) -> str: settings = self._get_settings() return ( f"{self._host_url}/prompts/{prompt_name}/{commit_hash[:8]}" - f"?organizationId={settings['id']}" + f"?organizationId={settings.id}" ) def _prompt_exists(self, prompt_identifier: str) -> bool: @@ -4875,7 +4875,7 @@ def create_prompt( *, description: Optional[str] = None, readme: Optional[str] = None, - tags: Optional[List[str]] = None, + tags: Optional[Sequence[str]] = None, is_public: bool = False, ) -> ls_schemas.Prompt: """Create a new prompt. @@ -4886,7 +4886,7 @@ def create_prompt( prompt_name (str): The name of the prompt. description (Optional[str]): A description of the prompt. readme (Optional[str]): A readme for the prompt. - tags (Optional[List[str]]): A list of tags for the prompt. + tags (Optional[Sequence[str]]): A list of tags for the prompt. is_public (bool): Whether the prompt should be public. Defaults to False. Returns: @@ -4897,7 +4897,7 @@ def create_prompt( HTTPError: If the server request fails. """ settings = self._get_settings() - if is_public and not settings.get("tenant_handle"): + if is_public and not settings.tenant_handle: raise ls_utils.LangSmithUserError( "Cannot create a public prompt without first\n" "creating a LangChain Hub handle. " @@ -4909,7 +4909,7 @@ def create_prompt( if not self._current_tenant_is_owner(owner=owner): raise self._owner_conflict_error("create a prompt", owner) - json: Dict[str, Union[str, bool, List[str]]] = { + json: Dict[str, Union[str, bool, Sequence[str]]] = { "repo_handle": prompt_name, "description": description or "", "readme": readme or "", @@ -4926,7 +4926,7 @@ def create_commit( prompt_identifier: str, object: Any, *, - parent_commit_hash: Optional[str] = "latest", + parent_commit_hash: Optional[str] = None, ) -> str: """Create a commit for an existing prompt. @@ -4934,7 +4934,7 @@ def create_commit( prompt_identifier (str): The identifier of the prompt. object (Any): The LangChain object to commit. parent_commit_hash (Optional[str]): The hash of the parent commit. - Defaults to "latest". + Defaults to latest commit. Returns: str: The url of the prompt commit. @@ -4962,7 +4962,7 @@ def create_commit( owner, prompt_name, _ = ls_utils.parse_prompt_identifier(prompt_identifier) prompt_owner_and_name = f"{owner}/{prompt_name}" - if parent_commit_hash == "latest": + if parent_commit_hash == "latest" or parent_commit_hash is None: parent_commit_hash = self._get_latest_commit_hash(prompt_owner_and_name) request_dict = {"parent_commit": parent_commit_hash, "manifest": manifest_dict} @@ -4980,7 +4980,7 @@ def update_prompt( *, description: Optional[str] = None, readme: Optional[str] = None, - tags: Optional[List[str]] = None, + tags: Optional[Sequence[str]] = None, is_public: Optional[bool] = None, is_archived: Optional[bool] = None, ) -> Dict[str, Any]: @@ -4992,7 +4992,7 @@ def update_prompt( prompt_identifier (str): The identifier of the prompt to update. description (Optional[str]): New description for the prompt. readme (Optional[str]): New readme for the prompt. - tags (Optional[List[str]]): New list of tags for the prompt. + tags (Optional[Sequence[str]]): New list of tags for the prompt. is_public (Optional[bool]): New public status for the prompt. is_archived (Optional[bool]): New archived status for the prompt. @@ -5004,7 +5004,7 @@ def update_prompt( HTTPError: If the server request fails. """ settings = self._get_settings() - if is_public and not settings.get("tenant_handle"): + if is_public and not settings.tenant_handle: raise ValueError( "Cannot create a public prompt without first\n" "creating a LangChain Hub handle. " @@ -5012,7 +5012,7 @@ def update_prompt( "https://smith.langchain.com/prompts" ) - json: Dict[str, Union[str, bool, List[str]]] = {} + json: Dict[str, Union[str, bool, Sequence[str]]] = {} if description is not None: json["description"] = description @@ -5158,11 +5158,11 @@ def push_prompt( prompt_identifier: str, *, object: Optional[Any] = None, - parent_commit_hash: Optional[str] = "latest", + parent_commit_hash: str = "latest", is_public: bool = False, - description: Optional[str] = "", - readme: Optional[str] = "", - tags: Optional[List[str]] = [], + description: Optional[str] = None, + readme: Optional[str] = None, + tags: Optional[Sequence[str]] = None, ) -> str: """Push a prompt to the LangSmith API. @@ -5174,14 +5174,14 @@ def push_prompt( Args: prompt_identifier (str): The identifier of the prompt. object (Optional[Any]): The LangChain object to push. - parent_commit_hash (Optional[str]): The parent commit hash. + parent_commit_hash (str): The parent commit hash. Defaults to "latest". is_public (bool): Whether the prompt should be public. Defaults to False. description (Optional[str]): A description of the prompt. Defaults to an empty string. readme (Optional[str]): A readme for the prompt. Defaults to an empty string. - tags (Optional[List[str]]): A list of tags for the prompt. + tags (Optional[Sequence[str]]): A list of tags for the prompt. Defaults to an empty list. Returns: @@ -5367,7 +5367,8 @@ def _tracing_sub_thread_func( def convert_prompt_to_openai_format( - messages: Any, stop: Optional[List[str]] = None, **kwargs: Any + messages: Any, + model_kwargs: Optional[Dict[str, Any]] = None, ) -> dict: """Convert a prompt to OpenAI format. @@ -5375,11 +5376,15 @@ def convert_prompt_to_openai_format( Args: messages (Any): The messages to convert. - stop (Optional[List[str]]): Stop sequences for the prompt. - **kwargs: Additional arguments for the conversion. + model_kwargs (Optional[Dict[str, Any]]): Model configuration arguments including + `stop` and any other required arguments. Defaults to None. Returns: dict: The prompt in OpenAI format. + + Raises: + ImportError: If the `langchain_openai` package is not installed. + ls_utils.LangSmithError: If there is an error during the conversion process. """ try: from langchain_openai import ChatOpenAI @@ -5391,17 +5396,18 @@ def convert_prompt_to_openai_format( openai = ChatOpenAI() + model_kwargs = model_kwargs or {} + stop = model_kwargs.pop("stop", None) + try: - return openai._get_request_payload(messages, stop=stop, **kwargs) + return openai._get_request_payload(messages, stop=stop, **model_kwargs) except Exception as e: raise ls_utils.LangSmithError(f"Error converting to OpenAI format: {e}") def convert_prompt_to_anthropic_format( messages: Any, - model_name: str = "claude-2", - stop: Optional[List[str]] = None, - **kwargs: Any, + model_kwargs: Optional[Dict[str, Any]] = None, ) -> dict: """Convert a prompt to Anthropic format. @@ -5409,9 +5415,9 @@ def convert_prompt_to_anthropic_format( Args: messages (Any): The messages to convert. - model_name (Optional[str]): The model name to use. Defaults to "claude-2". - stop (Optional[List[str]]): Stop sequences for the prompt. - **kwargs: Additional arguments for the conversion. + model_kwargs (Optional[Dict[str, Any]]): + Model configuration arguments including `model_name` and `stop`. + Defaults to None. Returns: dict: The prompt in Anthropic format. @@ -5425,11 +5431,16 @@ def convert_prompt_to_anthropic_format( "Install with `pip install langchain_anthropic`" ) + model_kwargs = model_kwargs or {} + model_name = model_kwargs.pop("model_name", "claude-3-haiku-20240307") + stop = model_kwargs.pop("stop", None) + timeout = model_kwargs.pop("timeout", None) + anthropic = ChatAnthropic( - model_name=model_name, timeout=None, stop=stop, base_url=None, api_key=None + model_name=model_name, timeout=timeout, stop=stop, **model_kwargs ) try: - return anthropic._get_request_payload(messages, stop=stop, **kwargs) + return anthropic._get_request_payload(messages, stop=stop) except Exception as e: raise ls_utils.LangSmithError(f"Error converting to Anthropic format: {e}") diff --git a/python/langsmith/schemas.py b/python/langsmith/schemas.py index f7eb3d955..1bf5787d9 100644 --- a/python/langsmith/schemas.py +++ b/python/langsmith/schemas.py @@ -667,6 +667,18 @@ class LangSmithInfo(BaseModel): Example.update_forward_refs() +class LangSmithSettings(BaseModel): + """Settings for the LangSmith tenant.""" + + id: str + """The ID of the tenant.""" + display_name: str + """The display name of the tenant.""" + created_at: datetime + """The creation time of the tenant.""" + tenant_handle: Optional[str] = None + + class FeedbackIngestToken(BaseModel): """Represents the schema for a feedback ingest token. diff --git a/python/tests/integration_tests/test_prompts.py b/python/tests/integration_tests/test_prompts.py index 538d7abd7..ed47244e0 100644 --- a/python/tests/integration_tests/test_prompts.py +++ b/python/tests/integration_tests/test_prompts.py @@ -150,7 +150,7 @@ def chat_prompt_template(): def test_current_tenant_is_owner(langsmith_client: Client): settings = langsmith_client._get_settings() - assert langsmith_client._current_tenant_is_owner(settings["tenant_handle"]) + assert langsmith_client._current_tenant_is_owner(settings.tenant_handle) assert langsmith_client._current_tenant_is_owner("-") assert not langsmith_client._current_tenant_is_owner("non_existent_owner") @@ -244,7 +244,7 @@ def test_pull_prompt(langsmith_client: Client, prompt_template_1: ChatPromptTemp assert pulled_prompt == pulled_prompt_2 # test pulling with tenant handle and name - tenant_handle = langsmith_client._get_settings()["tenant_handle"] + tenant_handle = langsmith_client._get_settings().tenant_handle pulled_prompt_3 = langsmith_client.pull_prompt(f"{tenant_handle}/{prompt_name}") assert pulled_prompt.metadata and pulled_prompt_3.metadata assert ( @@ -254,7 +254,7 @@ def test_pull_prompt(langsmith_client: Client, prompt_template_1: ChatPromptTemp assert pulled_prompt_3.metadata["lc_hub_owner"] == tenant_handle # test pulling with handle, name and commit hash - tenant_handle = langsmith_client._get_settings()["tenant_handle"] + tenant_handle = langsmith_client._get_settings().tenant_handle pulled_prompt_4 = langsmith_client.pull_prompt( f"{tenant_handle}/{prompt_name}:latest" ) @@ -545,7 +545,7 @@ def test_convert_to_anthropic_format(chat_prompt_template: ChatPromptTemplate): ) assert res == { - "model": "claude-2", + "model": "claude-3-haiku-20240307", "max_tokens": 1024, "messages": [{"role": "user", "content": "What is the meaning of life?"}], "system": "You are a chatbot", From 03328ec8eaeaef571df2b542fbedead68488a930 Mon Sep 17 00:00:00 2001 From: Maddy Adams Date: Tue, 16 Jul 2024 14:39:21 -0700 Subject: [PATCH 60/61] lint --- python/tests/integration_tests/test_prompts.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/python/tests/integration_tests/test_prompts.py b/python/tests/integration_tests/test_prompts.py index ed47244e0..6a669c299 100644 --- a/python/tests/integration_tests/test_prompts.py +++ b/python/tests/integration_tests/test_prompts.py @@ -150,7 +150,7 @@ def chat_prompt_template(): def test_current_tenant_is_owner(langsmith_client: Client): settings = langsmith_client._get_settings() - assert langsmith_client._current_tenant_is_owner(settings.tenant_handle) + assert langsmith_client._current_tenant_is_owner(settings.tenant_handle or "-") assert langsmith_client._current_tenant_is_owner("-") assert not langsmith_client._current_tenant_is_owner("non_existent_owner") @@ -540,12 +540,10 @@ def test_convert_to_openai_format(chat_prompt_template: ChatPromptTemplate): def test_convert_to_anthropic_format(chat_prompt_template: ChatPromptTemplate): invoked = chat_prompt_template.invoke({"question": "What is the meaning of life?"}) - res = convert_prompt_to_anthropic_format( - invoked, - ) + res = convert_prompt_to_anthropic_format(invoked, {"model_name": "claude-2"}) assert res == { - "model": "claude-3-haiku-20240307", + "model": "claude-2", "max_tokens": 1024, "messages": [{"role": "user", "content": "What is the meaning of life?"}], "system": "You are a chatbot", From af33f38d7ed84294fb329b6ac2e5bdc983713ead Mon Sep 17 00:00:00 2001 From: Maddy Adams Date: Tue, 16 Jul 2024 17:11:32 -0700 Subject: [PATCH 61/61] update version --- python/pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/pyproject.toml b/python/pyproject.toml index ec6123c5b..7efeed844 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "langsmith" -version = "0.1.86" +version = "0.1.88" description = "Client library to connect to the LangSmith LLM Tracing and Evaluation Platform." authors = ["LangChain "] license = "MIT"