diff --git a/.github/workflows/python_test.yml b/.github/workflows/python_test.yml index 5a45962ae..98020f18e 100644 --- a/.github/workflows/python_test.yml +++ b/.github/workflows/python_test.yml @@ -44,7 +44,7 @@ jobs: - name: Install dependencies run: | poetry install --with dev,lint - poetry run pip install -U langchain langchain-core + poetry run pip install -U langchain langchain-core langchain_anthropic langchain_openai - name: Build ${{ matrix.python-version }} run: poetry build - name: Lint ${{ matrix.python-version }} diff --git a/_scripts/_fetch_schema.py b/_scripts/_fetch_schema.py index 741e12a9c..ba8c171bd 100644 --- a/_scripts/_fetch_schema.py +++ b/_scripts/_fetch_schema.py @@ -1,4 +1,5 @@ """Fetch and prune the Langsmith spec.""" + import argparse from pathlib import Path @@ -19,7 +20,9 @@ def process_schema(sub_schema): get_dependencies(schema, sub_schema["$ref"].split("/")[-1], new_components) else: if "items" in sub_schema and "$ref" in sub_schema["items"]: - get_dependencies(schema, sub_schema["items"]["$ref"].split("/")[-1], new_components) + get_dependencies( + schema, sub_schema["items"]["$ref"].split("/")[-1], new_components + ) for keyword in ["anyOf", "oneOf", "allOf"]: if keyword in sub_schema: for item in sub_schema[keyword]: @@ -38,8 +41,6 @@ def process_schema(sub_schema): process_schema(item) - - def _extract_langsmith_routes_and_properties(schema, operation_ids): new_paths = {} new_components = {"schemas": {}} @@ -98,20 +99,25 @@ def test_openapi_specification(spec: dict): assert errors is None, f"OpenAPI validation failed: {errors}" -def main(out_file: str = "openapi.yaml", url: str = "https://web.smith.langchain.com/openapi.json"): +def main( + out_file: str = "openapi.yaml", + url: str = "https://web.smith.langchain.com/openapi.json", +): langsmith_schema = get_langsmith_runs_schema(url=url) parent_dir = Path(__file__).parent.parent test_openapi_specification(langsmith_schema) with (parent_dir / "openapi" / out_file).open("w") as f: # Sort the schema keys so the openapi version and info come at the top - for key in ['openapi', 'info', 'paths', 'components']: + for key in ["openapi", "info", "paths", "components"]: langsmith_schema[key] = langsmith_schema.pop(key) f.write(yaml.dump(langsmith_schema, sort_keys=False)) if __name__ == "__main__": parser = argparse.ArgumentParser() - parser.add_argument("--url", type=str, default="https://web.smith.langchain.com/openapi.json") + parser.add_argument( + "--url", type=str, default="https://web.smith.langchain.com/openapi.json" + ) parser.add_argument("--output", type=str, default="openapi.yaml") args = parser.parse_args() main(args.output, url=args.url) diff --git a/python/langsmith/client.py b/python/langsmith/client.py index 89644a16c..3ddcc9df0 100644 --- a/python/langsmith/client.py +++ b/python/langsmith/client.py @@ -472,6 +472,7 @@ class Client: "_hide_outputs", "_info", "_write_api_urls", + "_settings", ] def __init__( @@ -614,6 +615,8 @@ def __init__( else ls_utils.get_env_var("HIDE_OUTPUTS") == "true" ) + self._settings: Union[ls_schemas.LangSmithSettings, None] = None + def _repr_html_(self) -> str: """Return an HTML representation of the instance with a link to the URL. @@ -701,6 +704,19 @@ def info(self) -> ls_schemas.LangSmithInfo: self._info = ls_schemas.LangSmithInfo() return self._info + def _get_settings(self) -> ls_schemas.LangSmithSettings: + """Get the settings for the current tenant. + + Returns: + dict: The settings for the current tenant. + """ + if self._settings is None: + response = self.request_with_retries("GET", "/settings") + ls_utils.raise_for_status_with_text(response) + self._settings = ls_schemas.LangSmithSettings(**response.json()) + + return self._settings + def request_with_retries( self, /, @@ -4664,6 +4680,543 @@ def _evaluate_strings( **kwargs, ) + def _current_tenant_is_owner(self, owner: str) -> bool: + """Check if the current workspace has the same handle as owner. + + Args: + owner (str): The owner to check against. + + Returns: + bool: True if the current tenant is the owner, False otherwise. + """ + settings = self._get_settings() + return owner == "-" or settings.tenant_handle == owner + + def _owner_conflict_error( + self, action: str, owner: str + ) -> ls_utils.LangSmithUserError: + return ls_utils.LangSmithUserError( + f"Cannot {action} for another tenant.\n" + f"Current tenant: {self._get_settings().tenant_handle},\n" + f"Requested tenant: {owner}" + ) + + def _get_latest_commit_hash( + self, prompt_owner_and_name: str, limit: int = 1, offset: int = 0 + ) -> Optional[str]: + """Get the latest commit hash for a prompt. + + Args: + prompt_owner_and_name (str): The owner and name of the prompt. + limit (int): The maximum number of commits to fetch. Defaults to 1. + offset (int): The number of commits to skip. Defaults to 0. + + Returns: + Optional[str]: The latest commit hash, or None if no commits are found. + """ + response = self.request_with_retries( + "GET", + f"/commits/{prompt_owner_and_name}/", + params={"limit": limit, "offset": offset}, + ) + commits = response.json()["commits"] + return commits[0]["commit_hash"] if commits else None + + def _like_or_unlike_prompt( + self, prompt_identifier: str, like: bool + ) -> Dict[str, int]: + """Like or unlike a prompt. + + Args: + prompt_identifier (str): The identifier of the prompt. + like (bool): True to like the prompt, False to unlike it. + + Returns: + A dictionary with the key 'likes' and the count of likes as the value. + + Raises: + requests.exceptions.HTTPError: If the prompt is not found or + another error occurs. + """ + owner, prompt_name, _ = ls_utils.parse_prompt_identifier(prompt_identifier) + response = self.request_with_retries( + "POST", f"/likes/{owner}/{prompt_name}", json={"like": like} + ) + response.raise_for_status() + return response.json() + + def _get_prompt_url(self, prompt_identifier: str) -> str: + """Get a URL for a prompt. + + Args: + prompt_identifier (str): The identifier of the prompt. + + Returns: + str: The URL for the prompt. + + """ + owner, prompt_name, commit_hash = ls_utils.parse_prompt_identifier( + prompt_identifier + ) + + if not self._current_tenant_is_owner(owner): + return f"{self._host_url}/hub/{owner}/{prompt_name}:{commit_hash[:8]}" + + settings = self._get_settings() + return ( + f"{self._host_url}/prompts/{prompt_name}/{commit_hash[:8]}" + f"?organizationId={settings.id}" + ) + + def _prompt_exists(self, prompt_identifier: str) -> bool: + """Check if a prompt exists. + + Args: + prompt_identifier (str): The identifier of the prompt. + + Returns: + bool: True if the prompt exists, False otherwise. + """ + prompt = self.get_prompt(prompt_identifier) + return True if prompt else False + + def like_prompt(self, prompt_identifier: str) -> Dict[str, int]: + """Check if a prompt exists. + + Args: + prompt_identifier (str): The identifier of the prompt. + + Returns: + A dictionary with the key 'likes' and the count of likes as the value. + + """ + return self._like_or_unlike_prompt(prompt_identifier, like=True) + + def unlike_prompt(self, prompt_identifier: str) -> Dict[str, int]: + """Unlike a prompt. + + Args: + prompt_identifier (str): The identifier of the prompt. + + Returns: + A dictionary with the key 'likes' and the count of likes as the value. + + """ + return self._like_or_unlike_prompt(prompt_identifier, like=False) + + def list_prompts( + self, + *, + limit: int = 100, + offset: int = 0, + is_public: Optional[bool] = None, + is_archived: Optional[bool] = False, + sort_field: ls_schemas.PromptSortField = ls_schemas.PromptSortField.updated_at, + sort_direction: Literal["desc", "asc"] = "desc", + query: Optional[str] = None, + ) -> ls_schemas.ListPromptsResponse: + """List prompts with pagination. + + Args: + limit (int): The maximum number of prompts to return. Defaults to 100. + offset (int): The number of prompts to skip. Defaults to 0. + is_public (Optional[bool]): Filter prompts by if they are public. + is_archived (Optional[bool]): Filter prompts by if they are archived. + sort_field (ls_schemas.PromptsSortField): The field to sort by. + Defaults to "updated_at". + sort_direction (Literal["desc", "asc"]): The order to sort by. + Defaults to "desc". + query (Optional[str]): Filter prompts by a search query. + + Returns: + ls_schemas.ListPromptsResponse: A response object containing + the list of prompts. + """ + params = { + "limit": limit, + "offset": offset, + "is_public": ( + "true" if is_public else "false" if is_public is not None else None + ), + "is_archived": "true" if is_archived else "false", + "sort_field": sort_field, + "sort_direction": sort_direction, + "query": query, + "match_prefix": "true" if query else None, + } + + response = self.request_with_retries("GET", "/repos/", params=params) + return ls_schemas.ListPromptsResponse(**response.json()) + + def get_prompt(self, prompt_identifier: str) -> Optional[ls_schemas.Prompt]: + """Get a specific prompt by its identifier. + + Args: + prompt_identifier (str): The identifier of the prompt. + The identifier should be in the format "prompt_name" or "owner/prompt_name". + + Returns: + Optional[ls_schemas.Prompt]: The prompt object. + + Raises: + requests.exceptions.HTTPError: If the prompt is not found or + another error occurs. + """ + owner, prompt_name, _ = ls_utils.parse_prompt_identifier(prompt_identifier) + try: + response = self.request_with_retries("GET", f"/repos/{owner}/{prompt_name}") + return ls_schemas.Prompt(**response.json()["repo"]) + except ls_utils.LangSmithNotFoundError: + return None + + def create_prompt( + self, + prompt_identifier: str, + *, + description: Optional[str] = None, + readme: Optional[str] = None, + tags: Optional[Sequence[str]] = None, + is_public: bool = False, + ) -> ls_schemas.Prompt: + """Create a new prompt. + + Does not attach prompt object, just creates an empty prompt. + + Args: + prompt_name (str): The name of the prompt. + description (Optional[str]): A description of the prompt. + readme (Optional[str]): A readme for the prompt. + tags (Optional[Sequence[str]]): A list of tags for the prompt. + is_public (bool): Whether the prompt should be public. Defaults to False. + + Returns: + ls_schemas.Prompt: The created prompt object. + + Raises: + ValueError: If the current tenant is not the owner. + HTTPError: If the server request fails. + """ + settings = self._get_settings() + if is_public and not settings.tenant_handle: + raise ls_utils.LangSmithUserError( + "Cannot create a public prompt without first\n" + "creating a LangChain Hub handle. " + "You can add a handle by creating a public prompt at:\n" + "https://smith.langchain.com/prompts" + ) + + owner, prompt_name, _ = ls_utils.parse_prompt_identifier(prompt_identifier) + if not self._current_tenant_is_owner(owner=owner): + raise self._owner_conflict_error("create a prompt", owner) + + json: Dict[str, Union[str, bool, Sequence[str]]] = { + "repo_handle": prompt_name, + "description": description or "", + "readme": readme or "", + "tags": tags or [], + "is_public": is_public, + } + + response = self.request_with_retries("POST", "/repos/", json=json) + response.raise_for_status() + return ls_schemas.Prompt(**response.json()["repo"]) + + def create_commit( + self, + prompt_identifier: str, + object: Any, + *, + parent_commit_hash: Optional[str] = None, + ) -> str: + """Create a commit for an existing prompt. + + Args: + prompt_identifier (str): The identifier of the prompt. + object (Any): The LangChain object to commit. + parent_commit_hash (Optional[str]): The hash of the parent commit. + Defaults to latest commit. + + Returns: + str: The url of the prompt commit. + + Raises: + HTTPError: If the server request fails. + ValueError: If the prompt does not exist. + """ + if not self._prompt_exists(prompt_identifier): + raise ls_utils.LangSmithNotFoundError( + "Prompt does not exist, you must create it first." + ) + + try: + from langchain_core.load.dump import dumps + except ImportError: + raise ImportError( + "The client.create_commit function requires the langchain_core" + "package to run.\nInstall with `pip install langchain_core`" + ) + + json_object = dumps(object) + manifest_dict = json.loads(json_object) + + owner, prompt_name, _ = ls_utils.parse_prompt_identifier(prompt_identifier) + prompt_owner_and_name = f"{owner}/{prompt_name}" + + if parent_commit_hash == "latest" or parent_commit_hash is None: + parent_commit_hash = self._get_latest_commit_hash(prompt_owner_and_name) + + request_dict = {"parent_commit": parent_commit_hash, "manifest": manifest_dict} + response = self.request_with_retries( + "POST", f"/commits/{prompt_owner_and_name}", json=request_dict + ) + + commit_hash = response.json()["commit"]["commit_hash"] + + return self._get_prompt_url(f"{prompt_owner_and_name}:{commit_hash}") + + def update_prompt( + self, + prompt_identifier: str, + *, + description: Optional[str] = None, + readme: Optional[str] = None, + tags: Optional[Sequence[str]] = None, + is_public: Optional[bool] = None, + is_archived: Optional[bool] = None, + ) -> Dict[str, Any]: + """Update a prompt's metadata. + + To update the content of a prompt, use push_prompt or create_commit instead. + + Args: + prompt_identifier (str): The identifier of the prompt to update. + description (Optional[str]): New description for the prompt. + readme (Optional[str]): New readme for the prompt. + tags (Optional[Sequence[str]]): New list of tags for the prompt. + is_public (Optional[bool]): New public status for the prompt. + is_archived (Optional[bool]): New archived status for the prompt. + + Returns: + Dict[str, Any]: The updated prompt data as returned by the server. + + Raises: + ValueError: If the prompt_identifier is empty. + HTTPError: If the server request fails. + """ + settings = self._get_settings() + if is_public and not settings.tenant_handle: + raise ValueError( + "Cannot create a public prompt without first\n" + "creating a LangChain Hub handle. " + "You can add a handle by creating a public prompt at:\n" + "https://smith.langchain.com/prompts" + ) + + json: Dict[str, Union[str, bool, Sequence[str]]] = {} + + if description is not None: + json["description"] = description + if readme is not None: + json["readme"] = readme + if is_public is not None: + json["is_public"] = is_public + if is_archived is not None: + json["is_archived"] = is_archived + if tags is not None: + json["tags"] = tags + + owner, prompt_name, _ = ls_utils.parse_prompt_identifier(prompt_identifier) + response = self.request_with_retries( + "PATCH", f"/repos/{owner}/{prompt_name}", json=json + ) + response.raise_for_status() + return response.json() + + def delete_prompt(self, prompt_identifier: str) -> None: + """Delete a prompt. + + Args: + prompt_identifier (str): The identifier of the prompt to delete. + + Returns: + bool: True if the prompt was successfully deleted, False otherwise. + + Raises: + ValueError: If the current tenant is not the owner of the prompt. + """ + owner, prompt_name, _ = ls_utils.parse_prompt_identifier(prompt_identifier) + if not self._current_tenant_is_owner(owner): + raise self._owner_conflict_error("delete a prompt", owner) + + response = self.request_with_retries("DELETE", f"/repos/{owner}/{prompt_name}") + response.raise_for_status() + + def pull_prompt_commit( + self, + prompt_identifier: str, + *, + include_model: Optional[bool] = False, + ) -> ls_schemas.PromptCommit: + """Pull a prompt object from the LangSmith API. + + Args: + prompt_identifier (str): The identifier of the prompt. + + Returns: + ls_schemas.PromptObject: The prompt object. + + Raises: + ValueError: If no commits are found for the prompt. + """ + owner, prompt_name, commit_hash = ls_utils.parse_prompt_identifier( + prompt_identifier + ) + use_optimization = ls_utils.is_version_greater_or_equal( + self.info.version, "0.5.23" + ) + + if not use_optimization and commit_hash == "latest": + latest_commit_hash = self._get_latest_commit_hash(f"{owner}/{prompt_name}") + if latest_commit_hash is None: + raise ValueError("No commits found") + else: + commit_hash = latest_commit_hash + + response = self.request_with_retries( + "GET", + ( + f"/commits/{owner}/{prompt_name}/{commit_hash}" + f"{'?include_model=true' if include_model else ''}" + ), + ) + return ls_schemas.PromptCommit( + **{"owner": owner, "repo": prompt_name, **response.json()} + ) + + def pull_prompt( + self, prompt_identifier: str, *, include_model: Optional[bool] = False + ) -> Any: + """Pull a prompt and return it as a LangChain PromptTemplate. + + This method requires `langchain_core`. + + Args: + prompt_identifier (str): The identifier of the prompt. + + Returns: + Any: The prompt object in the specified format. + """ + try: + from langchain_core.load.load import loads + from langchain_core.prompts import BasePromptTemplate + from langchain_core.runnables.base import RunnableSequence + except ImportError: + raise ImportError( + "The client.pull_prompt function requires the langchain_core" + "package to run.\nInstall with `pip install langchain_core`" + ) + + prompt_object = self.pull_prompt_commit( + prompt_identifier, include_model=include_model + ) + prompt = loads(json.dumps(prompt_object.manifest)) + + if ( + isinstance(prompt, BasePromptTemplate) + or isinstance(prompt, RunnableSequence) + and isinstance(prompt.first, BasePromptTemplate) + ): + prompt_template = ( + prompt + if isinstance(prompt, BasePromptTemplate) + else ( + prompt.first + if isinstance(prompt, RunnableSequence) + and isinstance(prompt.first, BasePromptTemplate) + else None + ) + ) + if prompt_template is None: + raise ls_utils.LangSmithError( + "Prompt object is not a valid prompt template." + ) + + if prompt_template.metadata is None: + prompt_template.metadata = {} + prompt_template.metadata.update( + { + "lc_hub_owner": prompt_object.owner, + "lc_hub_repo": prompt_object.repo, + "lc_hub_commit_hash": prompt_object.commit_hash, + } + ) + + return prompt + + def push_prompt( + self, + prompt_identifier: str, + *, + object: Optional[Any] = None, + parent_commit_hash: str = "latest", + is_public: bool = False, + description: Optional[str] = None, + readme: Optional[str] = None, + tags: Optional[Sequence[str]] = None, + ) -> str: + """Push a prompt to the LangSmith API. + + Can be used to update prompt metadata or prompt content. + + If the prompt does not exist, it will be created. + If the prompt exists, it will be updated. + + Args: + prompt_identifier (str): The identifier of the prompt. + object (Optional[Any]): The LangChain object to push. + parent_commit_hash (str): The parent commit hash. + Defaults to "latest". + is_public (bool): Whether the prompt should be public. Defaults to False. + description (Optional[str]): A description of the prompt. + Defaults to an empty string. + readme (Optional[str]): A readme for the prompt. + Defaults to an empty string. + tags (Optional[Sequence[str]]): A list of tags for the prompt. + Defaults to an empty list. + + Returns: + str: The URL of the prompt. + + """ + # Create or update prompt metadata + if self._prompt_exists(prompt_identifier): + self.update_prompt( + prompt_identifier, + description=description, + readme=readme, + tags=tags, + is_public=is_public, + ) + else: + self.create_prompt( + prompt_identifier, + is_public=is_public, + description=description, + readme=readme, + tags=tags, + ) + + if object is None: + return self._get_prompt_url(prompt_identifier=prompt_identifier) + + # Create a commit with the new manifest + url = self.create_commit( + prompt_identifier, + object, + parent_commit_hash=parent_commit_hash, + ) + return url + def _tracing_thread_drain_queue( tracing_queue: Queue, limit: int = 100, block: bool = True @@ -4811,3 +5364,83 @@ def _tracing_sub_thread_func( tracing_queue, limit=size_limit, block=False ): _tracing_thread_handle_batch(client, tracing_queue, next_batch) + + +def convert_prompt_to_openai_format( + messages: Any, + model_kwargs: Optional[Dict[str, Any]] = None, +) -> dict: + """Convert a prompt to OpenAI format. + + Requires the `langchain_openai` package to be installed. + + Args: + messages (Any): The messages to convert. + model_kwargs (Optional[Dict[str, Any]]): Model configuration arguments including + `stop` and any other required arguments. Defaults to None. + + Returns: + dict: The prompt in OpenAI format. + + Raises: + ImportError: If the `langchain_openai` package is not installed. + ls_utils.LangSmithError: If there is an error during the conversion process. + """ + try: + from langchain_openai import ChatOpenAI + except ImportError: + raise ImportError( + "The convert_prompt_to_openai_format function requires the langchain_openai" + "package to run.\nInstall with `pip install langchain_openai`" + ) + + openai = ChatOpenAI() + + model_kwargs = model_kwargs or {} + stop = model_kwargs.pop("stop", None) + + try: + return openai._get_request_payload(messages, stop=stop, **model_kwargs) + except Exception as e: + raise ls_utils.LangSmithError(f"Error converting to OpenAI format: {e}") + + +def convert_prompt_to_anthropic_format( + messages: Any, + model_kwargs: Optional[Dict[str, Any]] = None, +) -> dict: + """Convert a prompt to Anthropic format. + + Requires the `langchain_anthropic` package to be installed. + + Args: + messages (Any): The messages to convert. + model_kwargs (Optional[Dict[str, Any]]): + Model configuration arguments including `model_name` and `stop`. + Defaults to None. + + Returns: + dict: The prompt in Anthropic format. + """ + try: + from langchain_anthropic import ChatAnthropic + except ImportError: + raise ImportError( + "The convert_prompt_to_anthropic_format function requires the " + "langchain_anthropic package to run.\n" + "Install with `pip install langchain_anthropic`" + ) + + model_kwargs = model_kwargs or {} + model_name = model_kwargs.pop("model_name", "claude-3-haiku-20240307") + stop = model_kwargs.pop("stop", None) + timeout = model_kwargs.pop("timeout", None) + + anthropic = ChatAnthropic( + model_name=model_name, timeout=timeout, stop=stop, **model_kwargs + ) + + try: + return anthropic._get_request_payload(messages, stop=stop) + except Exception as e: + raise ls_utils.LangSmithError(f"Error converting to Anthropic format: {e}") diff --git a/python/langsmith/schemas.py b/python/langsmith/schemas.py index 453aa13de..1bf5787d9 100644 --- a/python/langsmith/schemas.py +++ b/python/langsmith/schemas.py @@ -667,6 +667,18 @@ class LangSmithInfo(BaseModel): Example.update_forward_refs() +class LangSmithSettings(BaseModel): + """Settings for the LangSmith tenant.""" + + id: str + """The ID of the tenant.""" + display_name: str + """The display name of the tenant.""" + created_at: datetime + """The creation time of the tenant.""" + tenant_handle: Optional[str] = None + + class FeedbackIngestToken(BaseModel): """Represents the schema for a feedback ingest token. @@ -744,3 +756,97 @@ def metadata(self) -> dict[str, Any]: if self.extra is None or "metadata" not in self.extra: return {} return self.extra["metadata"] + + +class PromptCommit(BaseModel): + """Represents a Prompt with a manifest. + + Attributes: + owner (str): The handle of the owner of the prompt. + repo (str): The name of the prompt. + commit_hash (str): The commit hash of the prompt. + manifest (Dict[str, Any]): The manifest of the prompt. + examples (List[dict]): The list of examples. + """ + + owner: str + """The handle of the owner of the prompt.""" + repo: str + """The name of the prompt.""" + commit_hash: str + """The commit hash of the prompt.""" + manifest: Dict[str, Any] + """The manifest of the prompt.""" + examples: List[dict] + """The list of examples.""" + + +class Prompt(BaseModel): + """Represents a Prompt with metadata.""" + + repo_handle: str + """The name of the prompt.""" + description: Optional[str] = None + """The description of the prompt.""" + readme: Optional[str] = None + """The README of the prompt.""" + id: str + """The ID of the prompt.""" + tenant_id: str + """The tenant ID of the prompt owner.""" + created_at: datetime + """The creation time of the prompt.""" + updated_at: datetime + """The last update time of the prompt.""" + is_public: bool + """Whether the prompt is public.""" + is_archived: bool + """Whether the prompt is archived.""" + tags: List[str] + """The tags associated with the prompt.""" + original_repo_id: Optional[str] = None + """The ID of the original prompt, if forked.""" + upstream_repo_id: Optional[str] = None + """The ID of the upstream prompt, if forked.""" + owner: Optional[str] + """The handle of the owner of the prompt.""" + full_name: str + """The full name of the prompt. (owner + repo_handle)""" + num_likes: int + """The number of likes.""" + num_downloads: int + """The number of downloads.""" + num_views: int + """The number of views.""" + liked_by_auth_user: bool + """Whether the prompt is liked by the authenticated user.""" + last_commit_hash: Optional[str] = None + """The hash of the last commit.""" + num_commits: int + """The number of commits.""" + original_repo_full_name: Optional[str] = None + """The full name of the original prompt, if forked.""" + upstream_repo_full_name: Optional[str] = None + """The full name of the upstream prompt, if forked.""" + + +class ListPromptsResponse(BaseModel): + """A list of prompts with metadata.""" + + repos: List[Prompt] + """The list of prompts.""" + total: int + """The total number of prompts.""" + + +class PromptSortField(str, Enum): + """Enum for sorting fields for prompts.""" + + num_downloads = "num_downloads" + """Number of downloads.""" + num_views = "num_views" + """Number of views.""" + updated_at = "updated_at" + """Last updated time.""" + num_likes = "num_likes" + """Number of likes.""" diff --git a/python/langsmith/utils.py b/python/langsmith/utils.py index d35558f4f..2456af92a 100644 --- a/python/langsmith/utils.py +++ b/python/langsmith/utils.py @@ -571,6 +571,50 @@ def deepish_copy(val: T) -> T: return _middle_copy(val, memo) +def is_version_greater_or_equal(current_version: str, target_version: str) -> bool: + """Check if the current version is greater or equal to the target version.""" + from packaging import version + + current = version.parse(current_version) + target = version.parse(target_version) + return current >= target + + +def parse_prompt_identifier(identifier: str) -> Tuple[str, str, str]: + """Parse a string in the format of owner/name:hash, name:hash, owner/name, or name. + + Args: + identifier (str): The prompt identifier to parse. + + Returns: + Tuple[str, str, str]: A tuple containing (owner, name, hash). + + Raises: + ValueError: If the identifier doesn't match the expected formats. + """ + if ( + not identifier + or identifier.count("/") > 1 + or identifier.startswith("/") + or identifier.endswith("/") + ): + raise ValueError(f"Invalid identifier format: {identifier}") + + parts = identifier.split(":", 1) + owner_name = parts[0] + commit = parts[1] if len(parts) > 1 else "latest" + + if "/" in owner_name: + owner, name = owner_name.split("/", 1) + if not owner or not name: + raise ValueError(f"Invalid identifier format: {identifier}") + return owner, name, commit + else: + if not owner_name: + raise ValueError(f"Invalid identifier format: {identifier}") + return "-", owner_name, commit + + P = ParamSpec("P") diff --git a/python/pyproject.toml b/python/pyproject.toml index 6a7e7cb10..7efeed844 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "langsmith" -version = "0.1.87" +version = "0.1.88" description = "Client library to connect to the LangSmith LLM Tracing and Evaluation Platform." authors = ["LangChain "] license = "MIT" diff --git a/python/tests/integration_tests/test_prompts.py b/python/tests/integration_tests/test_prompts.py new file mode 100644 index 000000000..6a669c299 --- /dev/null +++ b/python/tests/integration_tests/test_prompts.py @@ -0,0 +1,550 @@ +from typing import Literal +from uuid import uuid4 + +import pytest +from langchain_core.prompts import ( + BasePromptTemplate, + ChatPromptTemplate, + PromptTemplate, +) +from langchain_core.runnables.base import RunnableSequence + +import langsmith.schemas as ls_schemas +import langsmith.utils as ls_utils +from langsmith.client import ( + Client, + convert_prompt_to_anthropic_format, + convert_prompt_to_openai_format, +) + + +@pytest.fixture +def langsmith_client() -> Client: + return Client(timeout_ms=(50_000, 90_000)) + + +@pytest.fixture +def prompt_template_1() -> ChatPromptTemplate: + return ChatPromptTemplate.from_template("tell me a joke about {topic}") + + +@pytest.fixture +def prompt_template_2() -> ChatPromptTemplate: + return ChatPromptTemplate.from_messages( + [ + ("system", "You are a helpful assistant."), + ("human", "{question}"), + ] + ) + + +@pytest.fixture +def prompt_template_3() -> PromptTemplate: + return PromptTemplate.from_template("Summarize the following text: {text}") + + +@pytest.fixture +def prompt_with_model() -> dict: + return { + "id": ["langsmith", "playground", "PromptPlayground"], + "lc": 1, + "type": "constructor", + "kwargs": { + "last": { + "id": ["langchain", "schema", "runnable", "RunnableBinding"], + "lc": 1, + "type": "constructor", + "kwargs": { + "bound": { + "id": ["langchain", "chat_models", "openai", "ChatOpenAI"], + "lc": 1, + "type": "constructor", + "kwargs": { + "openai_api_key": { + "id": ["OPENAI_API_KEY"], + "lc": 1, + "type": "secret", + } + }, + }, + "kwargs": {}, + }, + }, + "first": { + "id": ["langchain", "prompts", "chat", "ChatPromptTemplate"], + "lc": 1, + "type": "constructor", + "kwargs": { + "messages": [ + { + "id": [ + "langchain", + "prompts", + "chat", + "SystemMessagePromptTemplate", + ], + "lc": 1, + "type": "constructor", + "kwargs": { + "prompt": { + "id": [ + "langchain", + "prompts", + "prompt", + "PromptTemplate", + ], + "lc": 1, + "type": "constructor", + "kwargs": { + "template": "You are a chatbot.", + "input_variables": [], + "template_format": "f-string", + }, + } + }, + }, + { + "id": [ + "langchain", + "prompts", + "chat", + "HumanMessagePromptTemplate", + ], + "lc": 1, + "type": "constructor", + "kwargs": { + "prompt": { + "id": [ + "langchain", + "prompts", + "prompt", + "PromptTemplate", + ], + "lc": 1, + "type": "constructor", + "kwargs": { + "template": "{question}", + "input_variables": ["question"], + "template_format": "f-string", + }, + } + }, + }, + ], + "input_variables": ["question"], + }, + }, + }, + } + + +@pytest.fixture +def chat_prompt_template(): + return ChatPromptTemplate.from_messages( + [ + ("system", "You are a chatbot"), + ("user", "{question}"), + ] + ) + + +def test_current_tenant_is_owner(langsmith_client: Client): + settings = langsmith_client._get_settings() + assert langsmith_client._current_tenant_is_owner(settings.tenant_handle or "-") + assert langsmith_client._current_tenant_is_owner("-") + assert not langsmith_client._current_tenant_is_owner("non_existent_owner") + + +def test_list_prompts(langsmith_client: Client): + response = langsmith_client.list_prompts(limit=10, offset=0) + assert isinstance(response, ls_schemas.ListPromptsResponse) + assert len(response.repos) <= 10 + + +def test_get_prompt(langsmith_client: Client, prompt_template_1: ChatPromptTemplate): + prompt_name = f"test_prompt_{uuid4().hex[:8]}" + url = langsmith_client.push_prompt(prompt_name, object=prompt_template_1) + assert isinstance(url, str) + assert langsmith_client._prompt_exists(prompt_name) + + prompt = langsmith_client.get_prompt(prompt_name) + assert isinstance(prompt, ls_schemas.Prompt) + assert prompt.repo_handle == prompt_name + + langsmith_client.delete_prompt(prompt_name) + + +def test_prompt_exists(langsmith_client: Client, prompt_template_2: ChatPromptTemplate): + non_existent_prompt = f"non_existent_{uuid4().hex[:8]}" + assert not langsmith_client._prompt_exists(non_existent_prompt) + + existent_prompt = f"existent_{uuid4().hex[:8]}" + assert langsmith_client.push_prompt(existent_prompt, object=prompt_template_2) + assert langsmith_client._prompt_exists(existent_prompt) + + langsmith_client.delete_prompt(existent_prompt) + + +def test_update_prompt(langsmith_client: Client, prompt_template_1: ChatPromptTemplate): + prompt_name = f"test_prompt_{uuid4().hex[:8]}" + langsmith_client.push_prompt(prompt_name, object=prompt_template_1) + + updated_data = langsmith_client.update_prompt( + prompt_name, + description="Updated description", + is_public=True, + tags=["test", "update"], + ) + assert isinstance(updated_data, dict) + + updated_prompt = langsmith_client.get_prompt(prompt_name) + assert isinstance(updated_prompt, ls_schemas.Prompt) + assert updated_prompt.description == "Updated description" + assert updated_prompt.is_public + assert set(updated_prompt.tags) == set(["test", "update"]) + + langsmith_client.delete_prompt(prompt_name) + + +def test_delete_prompt(langsmith_client: Client, prompt_template_1: ChatPromptTemplate): + prompt_name = f"test_prompt_{uuid4().hex[:8]}" + langsmith_client.push_prompt(prompt_name, object=prompt_template_1) + + assert langsmith_client._prompt_exists(prompt_name) + langsmith_client.delete_prompt(prompt_name) + assert not langsmith_client._prompt_exists(prompt_name) + + +def test_pull_prompt_object( + langsmith_client: Client, prompt_template_1: ChatPromptTemplate +): + prompt_name = f"test_prompt_{uuid4().hex[:8]}" + langsmith_client.push_prompt(prompt_name, object=prompt_template_1) + + manifest = langsmith_client.pull_prompt_commit(prompt_name) + assert isinstance(manifest, ls_schemas.PromptCommit) + assert manifest.repo == prompt_name + + langsmith_client.delete_prompt(prompt_name) + + +def test_pull_prompt(langsmith_client: Client, prompt_template_1: ChatPromptTemplate): + prompt_name = f"test_prompt_{uuid4().hex[:8]}" + langsmith_client.push_prompt(prompt_name, object=prompt_template_1) + + # test pulling with just prompt name + pulled_prompt = langsmith_client.pull_prompt(prompt_name) + assert isinstance(pulled_prompt, ChatPromptTemplate) + assert ( + pulled_prompt.metadata and pulled_prompt.metadata["lc_hub_repo"] == prompt_name + ) + + # test pulling with private owner (-) and name + pulled_prompt_2 = langsmith_client.pull_prompt(f"-/{prompt_name}") + assert pulled_prompt == pulled_prompt_2 + + # test pulling with tenant handle and name + tenant_handle = langsmith_client._get_settings().tenant_handle + pulled_prompt_3 = langsmith_client.pull_prompt(f"{tenant_handle}/{prompt_name}") + assert pulled_prompt.metadata and pulled_prompt_3.metadata + assert ( + pulled_prompt.metadata["lc_hub_commit_hash"] + == pulled_prompt_3.metadata["lc_hub_commit_hash"] + ) + assert pulled_prompt_3.metadata["lc_hub_owner"] == tenant_handle + + # test pulling with handle, name and commit hash + tenant_handle = langsmith_client._get_settings().tenant_handle + pulled_prompt_4 = langsmith_client.pull_prompt( + f"{tenant_handle}/{prompt_name}:latest" + ) + assert pulled_prompt_3 == pulled_prompt_4 + + # test pulling without handle, with commit hash + assert pulled_prompt_4.metadata + pulled_prompt_5 = langsmith_client.pull_prompt( + f"{prompt_name}:{pulled_prompt_4.metadata['lc_hub_commit_hash']}" + ) + assert pulled_prompt_5.metadata + assert ( + pulled_prompt_4.metadata["lc_hub_commit_hash"] + == pulled_prompt_5.metadata["lc_hub_commit_hash"] + ) + + langsmith_client.delete_prompt(prompt_name) + + +def test_push_and_pull_prompt( + langsmith_client: Client, prompt_template_2: ChatPromptTemplate +): + prompt_name = f"test_prompt_{uuid4().hex[:8]}" + + push_result = langsmith_client.push_prompt(prompt_name, object=prompt_template_2) + assert isinstance(push_result, str) + + pulled_prompt = langsmith_client.pull_prompt(prompt_name) + assert isinstance(pulled_prompt, ChatPromptTemplate) + + langsmith_client.delete_prompt(prompt_name) + + # should fail + with pytest.raises(ls_utils.LangSmithUserError): + langsmith_client.push_prompt( + f"random_handle/{prompt_name}", object=prompt_template_2 + ) + + +def test_pull_prompt_include_model(langsmith_client: Client, prompt_with_model: dict): + prompt_name = f"test_prompt_with_model_{uuid4().hex[:8]}" + langsmith_client.push_prompt(prompt_name, object=prompt_with_model) + + pulled_prompt = langsmith_client.pull_prompt(prompt_name, include_model=True) + assert isinstance(pulled_prompt, RunnableSequence) + if getattr(pulled_prompt, "first", None): + first = getattr(pulled_prompt, "first") + assert isinstance(first, BasePromptTemplate) + assert first.metadata and first.metadata["lc_hub_repo"] == prompt_name + else: + assert False, "pulled_prompt.first should exist, incorrect prompt format" + + langsmith_client.delete_prompt(prompt_name) + + +def test_like_unlike_prompt( + langsmith_client: Client, prompt_template_1: ChatPromptTemplate +): + prompt_name = f"test_prompt_{uuid4().hex[:8]}" + langsmith_client.push_prompt(prompt_name, object=prompt_template_1) + + langsmith_client.like_prompt(prompt_name) + prompt = langsmith_client.get_prompt(prompt_name) + assert isinstance(prompt, ls_schemas.Prompt) + assert prompt.num_likes == 1 + + langsmith_client.unlike_prompt(prompt_name) + prompt = langsmith_client.get_prompt(prompt_name) + assert isinstance(prompt, ls_schemas.Prompt) + assert prompt.num_likes == 0 + + langsmith_client.delete_prompt(prompt_name) + + +def test_get_latest_commit_hash( + langsmith_client: Client, prompt_template_1: ChatPromptTemplate +): + prompt_name = f"test_prompt_{uuid4().hex[:8]}" + langsmith_client.push_prompt(prompt_name, object=prompt_template_1) + + commit_hash = langsmith_client._get_latest_commit_hash(f"-/{prompt_name}") + assert isinstance(commit_hash, str) + assert len(commit_hash) > 0 + + langsmith_client.delete_prompt(prompt_name) + + +def test_create_prompt(langsmith_client: Client): + prompt_name = f"test_create_prompt_{uuid4().hex[:8]}" + created_prompt = langsmith_client.create_prompt( + prompt_name, + description="Test description", + readme="Test readme", + tags=["test", "create"], + is_public=False, + ) + assert isinstance(created_prompt, ls_schemas.Prompt) + assert created_prompt.repo_handle == prompt_name + assert created_prompt.description == "Test description" + assert created_prompt.readme == "Test readme" + assert set(created_prompt.tags) == set(["test", "create"]) + assert not created_prompt.is_public + + langsmith_client.delete_prompt(prompt_name) + + +def test_create_commit( + langsmith_client: Client, + prompt_template_2: ChatPromptTemplate, + prompt_template_3: PromptTemplate, +): + prompt_name = f"test_create_commit_{uuid4().hex[:8]}" + try: + # this should fail because the prompt does not exist + commit_url = langsmith_client.create_commit( + prompt_name, object=prompt_template_2 + ) + pytest.fail("Expected LangSmithNotFoundError was not raised") + except ls_utils.LangSmithNotFoundError as e: + assert str(e) == "Prompt does not exist, you must create it first." + except Exception as e: + pytest.fail(f"Unexpected exception raised: {e}") + + langsmith_client.push_prompt(prompt_name, object=prompt_template_3) + commit_url = langsmith_client.create_commit(prompt_name, object=prompt_template_2) + assert isinstance(commit_url, str) + assert prompt_name in commit_url + + prompt = langsmith_client.get_prompt(prompt_name) + assert isinstance(prompt, ls_schemas.Prompt) + assert prompt.num_commits == 2 + + # try submitting different types of unaccepted manifests + try: + # this should fail + commit_url = langsmith_client.create_commit(prompt_name, object={"hi": "hello"}) + except ls_utils.LangSmithError as e: + err = str(e) + assert "Manifest must have an id field" in err + assert "400 Client Error" in err + except Exception as e: + pytest.fail(f"Unexpected exception raised: {e}") + + try: + # this should fail + commit_url = langsmith_client.create_commit(prompt_name, object={"id": ["hi"]}) + except ls_utils.LangSmithError as e: + err = str(e) + assert "Manifest type hi is not supported" in err + assert "400 Client Error" in err + except Exception as e: + pytest.fail(f"Unexpected exception raised: {e}") + + langsmith_client.delete_prompt(prompt_name) + + +def test_push_prompt(langsmith_client: Client, prompt_template_3: PromptTemplate): + prompt_name = f"test_push_new_{uuid4().hex[:8]}" + url = langsmith_client.push_prompt( + prompt_name, + object=prompt_template_3, + is_public=True, + description="New prompt", + tags=["new", "test"], + ) + + assert isinstance(url, str) + assert prompt_name in url + + prompt = langsmith_client.get_prompt(prompt_name) + assert isinstance(prompt, ls_schemas.Prompt) + assert prompt.is_public + assert prompt.description == "New prompt" + assert "new" in prompt.tags + assert "test" in prompt.tags + assert prompt.num_commits == 1 + + # test updating prompt metadata but not manifest + url = langsmith_client.push_prompt( + prompt_name, + is_public=False, + description="Updated prompt", + ) + + updated_prompt = langsmith_client.get_prompt(prompt_name) + assert isinstance(updated_prompt, ls_schemas.Prompt) + assert updated_prompt.description == "Updated prompt" + assert not updated_prompt.is_public + assert updated_prompt.num_commits == 1 + + langsmith_client.delete_prompt(prompt_name) + + +@pytest.mark.parametrize("is_public,expected_count", [(True, 1), (False, 1)]) +def test_list_prompts_filter( + langsmith_client: Client, + prompt_template_1: ChatPromptTemplate, + is_public: bool, + expected_count: int, +): + prompt_name = f"test_list_filter_{uuid4().hex[:8]}" + langsmith_client.push_prompt( + prompt_name, object=prompt_template_1, is_public=is_public + ) + + response = langsmith_client.list_prompts(is_public=is_public, query=prompt_name) + + assert response.total == expected_count + if expected_count > 0: + assert response.repos[0].repo_handle == prompt_name + + langsmith_client.delete_prompt(prompt_name) + + +def test_update_prompt_archive( + langsmith_client: Client, prompt_template_1: ChatPromptTemplate +): + prompt_name = f"test_archive_{uuid4().hex[:8]}" + langsmith_client.push_prompt(prompt_name, object=prompt_template_1) + + langsmith_client.update_prompt(prompt_name, is_archived=True) + archived_prompt = langsmith_client.get_prompt(prompt_name) + assert isinstance(archived_prompt, ls_schemas.Prompt) + assert archived_prompt.is_archived + + langsmith_client.update_prompt(prompt_name, is_archived=False) + unarchived_prompt = langsmith_client.get_prompt(prompt_name) + assert isinstance(unarchived_prompt, ls_schemas.Prompt) + assert not unarchived_prompt.is_archived + + langsmith_client.delete_prompt(prompt_name) + + +@pytest.mark.parametrize( + "sort_field, sort_direction", + [ + (ls_schemas.PromptSortField.updated_at, "desc"), + ], +) +def test_list_prompts_sorting( + langsmith_client: Client, + prompt_template_1: ChatPromptTemplate, + sort_field: ls_schemas.PromptSortField, + sort_direction: Literal["asc", "desc"], +): + prompt_names = [f"test_sort_{i}_{uuid4().hex[:8]}" for i in range(3)] + for name in prompt_names: + langsmith_client.push_prompt(name, object=prompt_template_1) + + response = langsmith_client.list_prompts( + sort_field=sort_field, sort_direction=sort_direction, limit=10 + ) + + assert len(response.repos) >= 3 + sorted_names = [ + repo.repo_handle for repo in response.repos if repo.repo_handle in prompt_names + ] + assert sorted_names == sorted(sorted_names, reverse=(sort_direction == "desc")) + + for name in prompt_names: + langsmith_client.delete_prompt(name) + + +def test_convert_to_openai_format(chat_prompt_template: ChatPromptTemplate): + invoked = chat_prompt_template.invoke({"question": "What is the meaning of life?"}) + + res = convert_prompt_to_openai_format( + invoked, + ) + + assert res == { + "messages": [ + {"content": "You are a chatbot", "role": "system"}, + {"content": "What is the meaning of life?", "role": "user"}, + ], + "model": "gpt-3.5-turbo", + "stream": False, + "n": 1, + "temperature": 0.7, + } + + +def test_convert_to_anthropic_format(chat_prompt_template: ChatPromptTemplate): + invoked = chat_prompt_template.invoke({"question": "What is the meaning of life?"}) + + res = convert_prompt_to_anthropic_format(invoked, {"model_name": "claude-2"}) + + assert res == { + "model": "claude-2", + "max_tokens": 1024, + "messages": [{"role": "user", "content": "What is the meaning of life?"}], + "system": "You are a chatbot", + } diff --git a/python/tests/unit_tests/test_utils.py b/python/tests/unit_tests/test_utils.py index 9cadaa9cb..8fd493478 100644 --- a/python/tests/unit_tests/test_utils.py +++ b/python/tests/unit_tests/test_utils.py @@ -264,3 +264,50 @@ class MyNamedTuple(NamedTuple): "fake_json": ClassWithFakeJson(), } assert ls_utils.deepish_copy(my_dict) == my_dict + + +def test_is_version_greater_or_equal(): + # Test versions equal to 0.5.23 + assert ls_utils.is_version_greater_or_equal("0.5.23", "0.5.23") + + # Test versions greater than 0.5.23 + assert ls_utils.is_version_greater_or_equal("0.5.24", "0.5.23") + assert ls_utils.is_version_greater_or_equal("0.6.0", "0.5.23") + assert ls_utils.is_version_greater_or_equal("1.0.0", "0.5.23") + + # Test versions less than 0.5.23 + assert not ls_utils.is_version_greater_or_equal("0.5.22", "0.5.23") + assert not ls_utils.is_version_greater_or_equal("0.5.0", "0.5.23") + assert not ls_utils.is_version_greater_or_equal("0.4.99", "0.5.23") + + +def test_parse_prompt_identifier(): + # Valid cases + assert ls_utils.parse_prompt_identifier("name") == ("-", "name", "latest") + assert ls_utils.parse_prompt_identifier("owner/name") == ("owner", "name", "latest") + assert ls_utils.parse_prompt_identifier("owner/name:commit") == ( + "owner", + "name", + "commit", + ) + assert ls_utils.parse_prompt_identifier("name:commit") == ("-", "name", "commit") + + # Invalid cases + invalid_identifiers = [ + "", + "/", + ":", + "owner/", + "/name", + "owner//name", + "owner/name/", + "owner/name/extra", + ":commit", + ] + + for invalid_id in invalid_identifiers: + try: + ls_utils.parse_prompt_identifier(invalid_id) + assert False, f"Expected ValueError for identifier: {invalid_id}" + except ValueError: + pass # This is the expected behavior