From e2c4f50b169eed2d76499ef7588d9e089e357b89 Mon Sep 17 00:00:00 2001 From: Eugene Yurtsev Date: Wed, 6 Dec 2023 14:51:35 -0500 Subject: [PATCH] Example: Chat with history persisted on backend (#296) This PR shows a simple chat with history being persisted on the backend. It uses the file system to persist user chats. The example is meant to illustrate how to use the interfaces, should not be used as is in production setting. --- examples/chat_with_persistence/client.ipynb | 330 ++++++++++++++++++++ examples/chat_with_persistence/server.py | 163 ++++++++++ 2 files changed, 493 insertions(+) create mode 100644 examples/chat_with_persistence/client.ipynb create mode 100755 examples/chat_with_persistence/server.py diff --git a/examples/chat_with_persistence/client.ipynb b/examples/chat_with_persistence/client.ipynb new file mode 100644 index 00000000..85abb02b --- /dev/null +++ b/examples/chat_with_persistence/client.ipynb @@ -0,0 +1,330 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Chat History\n", + "\n", + "Here we'll be interacting with a server that's exposing a chat bot with message history being persisted on the backend." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "import uuid\n", + "from langserve import RemoteRunnable\n", + "\n", + "conversation_id = str(uuid.uuid4())\n", + "chat = RemoteRunnable(\"http://localhost:8000/\", cookies={\"user_id\": \"eugene\"})" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Let's create a prompt composed of a system message and a human message." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": { + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "AIMessage(content=' Hello Eugene! My name is Claude.')" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "chat.invoke({\"human_input\": \"my name is eugene. what is your name?\"}, {'configurable': { 'conversation_id': conversation_id } })" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": { + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "AIMessage(content=' You said your name is Eugene.')" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "chat.invoke({\"human_input\": \"what was my name?\"}, {'configurable': { 'conversation_id': conversation_id } })" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Use different user but same conversation id" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "chat = RemoteRunnable(\"http://localhost:8000/\", cookies={\"user_id\": \"nuno\"})" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": { + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "AIMessage(content=\" I'm afraid I don't actually know your name. Earlier I had randomly suggested you could imagine yourself being an assistant named Bob, but that was just for the sake of an example. I don't have any information about your actual name or identity.\")" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "chat.invoke({\"human_input\": \"what was my name?\"}, {'configurable': { 'conversation_id': conversation_id }})" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": { + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + " Sure\n", + ",\n", + " I\n", + "'d\n", + " be\n", + " happy\n", + " to\n", + " count\n", + " to\n", + " 10\n", + ":\n", + "\n", + "\n", + "1\n", + "\n", + "2\n", + "\n", + "3\n", + "\n", + "4\n", + "\n", + "5\n", + "\n", + "6\n", + "\n", + "7\n", + "\n", + "8\n", + "\n", + "9\n", + "\n", + "10\n" + ] + } + ], + "source": [ + "for chunk in chat.stream({'human_input': \"Can you count till 10?\"}, {'configurable': { 'conversation_id': conversation_id } }):\n", + " print()\n", + " print(chunk.content, end='', flush=True)" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[01;34mchat_histories/\u001b[0m\n", + "├── \u001b[01;34meugene\u001b[0m\n", + "│   └── ebd5f04f-6307-455a-b371-ecbae88ef8a9.json\n", + "└── \u001b[01;34mnuno\u001b[0m\n", + " └── ebd5f04f-6307-455a-b371-ecbae88ef8a9.json\n", + "\n", + "2 directories, 2 files\n" + ] + } + ], + "source": [ + "!tree chat_histories/" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": { + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[1;39m[\n", + " \u001b[1;39m{\n", + " \u001b[0m\u001b[34;1m\"type\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"human\"\u001b[0m\u001b[1;39m,\n", + " \u001b[0m\u001b[34;1m\"data\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m{\n", + " \u001b[0m\u001b[34;1m\"content\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"my name is eugene. what is your name?\"\u001b[0m\u001b[1;39m,\n", + " \u001b[0m\u001b[34;1m\"additional_kwargs\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m{}\u001b[0m\u001b[1;39m,\n", + " \u001b[0m\u001b[34;1m\"type\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"human\"\u001b[0m\u001b[1;39m,\n", + " \u001b[0m\u001b[34;1m\"example\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;39mfalse\u001b[0m\u001b[1;39m\n", + " \u001b[1;39m}\u001b[0m\u001b[1;39m\n", + " \u001b[1;39m}\u001b[0m\u001b[1;39m,\n", + " \u001b[1;39m{\n", + " \u001b[0m\u001b[34;1m\"type\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"ai\"\u001b[0m\u001b[1;39m,\n", + " \u001b[0m\u001b[34;1m\"data\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m{\n", + " \u001b[0m\u001b[34;1m\"content\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\" Hello Eugene! My name is Claude.\"\u001b[0m\u001b[1;39m,\n", + " \u001b[0m\u001b[34;1m\"additional_kwargs\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m{}\u001b[0m\u001b[1;39m,\n", + " \u001b[0m\u001b[34;1m\"type\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"ai\"\u001b[0m\u001b[1;39m,\n", + " \u001b[0m\u001b[34;1m\"example\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;39mfalse\u001b[0m\u001b[1;39m\n", + " \u001b[1;39m}\u001b[0m\u001b[1;39m\n", + " \u001b[1;39m}\u001b[0m\u001b[1;39m,\n", + " \u001b[1;39m{\n", + " \u001b[0m\u001b[34;1m\"type\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"human\"\u001b[0m\u001b[1;39m,\n", + " \u001b[0m\u001b[34;1m\"data\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m{\n", + " \u001b[0m\u001b[34;1m\"content\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"what was my name?\"\u001b[0m\u001b[1;39m,\n", + " \u001b[0m\u001b[34;1m\"additional_kwargs\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m{}\u001b[0m\u001b[1;39m,\n", + " \u001b[0m\u001b[34;1m\"type\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"human\"\u001b[0m\u001b[1;39m,\n", + " \u001b[0m\u001b[34;1m\"example\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;39mfalse\u001b[0m\u001b[1;39m\n", + " \u001b[1;39m}\u001b[0m\u001b[1;39m\n", + " \u001b[1;39m}\u001b[0m\u001b[1;39m,\n", + " \u001b[1;39m{\n", + " \u001b[0m\u001b[34;1m\"type\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"ai\"\u001b[0m\u001b[1;39m,\n", + " \u001b[0m\u001b[34;1m\"data\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m{\n", + " \u001b[0m\u001b[34;1m\"content\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\" You said your name is Eugene.\"\u001b[0m\u001b[1;39m,\n", + " \u001b[0m\u001b[34;1m\"additional_kwargs\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m{}\u001b[0m\u001b[1;39m,\n", + " \u001b[0m\u001b[34;1m\"type\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"ai\"\u001b[0m\u001b[1;39m,\n", + " \u001b[0m\u001b[34;1m\"example\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;39mfalse\u001b[0m\u001b[1;39m\n", + " \u001b[1;39m}\u001b[0m\u001b[1;39m\n", + " \u001b[1;39m}\u001b[0m\u001b[1;39m\n", + "\u001b[1;39m]\u001b[0m\n" + ] + } + ], + "source": [ + "!cat chat_histories/eugene/ebd5f04f-6307-455a-b371-ecbae88ef8a9.json | jq ." + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": { + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[1;39m[\n", + " \u001b[1;39m{\n", + " \u001b[0m\u001b[34;1m\"type\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"human\"\u001b[0m\u001b[1;39m,\n", + " \u001b[0m\u001b[34;1m\"data\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m{\n", + " \u001b[0m\u001b[34;1m\"content\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"what was my name?\"\u001b[0m\u001b[1;39m,\n", + " \u001b[0m\u001b[34;1m\"additional_kwargs\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m{}\u001b[0m\u001b[1;39m,\n", + " \u001b[0m\u001b[34;1m\"type\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"human\"\u001b[0m\u001b[1;39m,\n", + " \u001b[0m\u001b[34;1m\"example\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;39mfalse\u001b[0m\u001b[1;39m\n", + " \u001b[1;39m}\u001b[0m\u001b[1;39m\n", + " \u001b[1;39m}\u001b[0m\u001b[1;39m,\n", + " \u001b[1;39m{\n", + " \u001b[0m\u001b[34;1m\"type\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"ai\"\u001b[0m\u001b[1;39m,\n", + " \u001b[0m\u001b[34;1m\"data\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m{\n", + " \u001b[0m\u001b[34;1m\"content\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\" I'm afraid I don't actually know your name. Earlier I had randomly suggested you could imagine yourself being an assistant named Bob, but that was just for the sake of an example. I don't have any information about your actual name or identity.\"\u001b[0m\u001b[1;39m,\n", + " \u001b[0m\u001b[34;1m\"additional_kwargs\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m{}\u001b[0m\u001b[1;39m,\n", + " \u001b[0m\u001b[34;1m\"type\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"ai\"\u001b[0m\u001b[1;39m,\n", + " \u001b[0m\u001b[34;1m\"example\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;39mfalse\u001b[0m\u001b[1;39m\n", + " \u001b[1;39m}\u001b[0m\u001b[1;39m\n", + " \u001b[1;39m}\u001b[0m\u001b[1;39m,\n", + " \u001b[1;39m{\n", + " \u001b[0m\u001b[34;1m\"type\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"human\"\u001b[0m\u001b[1;39m,\n", + " \u001b[0m\u001b[34;1m\"data\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m{\n", + " \u001b[0m\u001b[34;1m\"content\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"Can you count till 10?\"\u001b[0m\u001b[1;39m,\n", + " \u001b[0m\u001b[34;1m\"additional_kwargs\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m{}\u001b[0m\u001b[1;39m,\n", + " \u001b[0m\u001b[34;1m\"type\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"human\"\u001b[0m\u001b[1;39m,\n", + " \u001b[0m\u001b[34;1m\"example\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;39mfalse\u001b[0m\u001b[1;39m\n", + " \u001b[1;39m}\u001b[0m\u001b[1;39m\n", + " \u001b[1;39m}\u001b[0m\u001b[1;39m,\n", + " \u001b[1;39m{\n", + " \u001b[0m\u001b[34;1m\"type\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"AIMessageChunk\"\u001b[0m\u001b[1;39m,\n", + " \u001b[0m\u001b[34;1m\"data\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m{\n", + " \u001b[0m\u001b[34;1m\"content\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\" Sure, I'd be happy to count to 10:\\n\\n1\\n2\\n3\\n4\\n5\\n6\\n7\\n8\\n9\\n10\"\u001b[0m\u001b[1;39m,\n", + " \u001b[0m\u001b[34;1m\"additional_kwargs\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m{}\u001b[0m\u001b[1;39m,\n", + " \u001b[0m\u001b[34;1m\"type\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"AIMessageChunk\"\u001b[0m\u001b[1;39m,\n", + " \u001b[0m\u001b[34;1m\"example\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;39mfalse\u001b[0m\u001b[1;39m\n", + " \u001b[1;39m}\u001b[0m\u001b[1;39m\n", + " \u001b[1;39m}\u001b[0m\u001b[1;39m\n", + "\u001b[1;39m]\u001b[0m\n" + ] + } + ], + "source": [ + "!cat chat_histories/nuno/ebd5f04f-6307-455a-b371-ecbae88ef8a9.json | jq ." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.6" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/examples/chat_with_persistence/server.py b/examples/chat_with_persistence/server.py new file mode 100755 index 00000000..61c73a4c --- /dev/null +++ b/examples/chat_with_persistence/server.py @@ -0,0 +1,163 @@ +#!/usr/bin/env python +"""Example of a chat server with persistence handled on the backend. + +For simplicity, we're using file storage here -- to avoid the need to set up +a database. This is obviously not a good idea for a production environment, +but will help us to demonstrate the RunnableWithMessageHistory interface. + +We'll use cookies to identify the user. This will help illustrate how to +fetch configuration from the request. +""" +import re +from pathlib import Path +from typing import Any, Callable, Dict, Union + +from fastapi import FastAPI, HTTPException, Request +from langchain.chat_models import ChatAnthropic +from langchain.memory import FileChatMessageHistory +from langchain.schema.runnable.utils import ConfigurableFieldSpec +from langchain_core.chat_history import BaseChatMessageHistory +from langchain_core.prompts import ChatPromptTemplate, MessagesPlaceholder +from langchain_core.runnables.history import RunnableWithMessageHistory +from typing_extensions import TypedDict + +from langserve import add_routes + + +def _is_valid_identifier(value: str) -> bool: + """Check if the session ID is in a valid format.""" + # Use a regular expression to match the allowed characters + valid_characters = re.compile(r"^[a-zA-Z0-9-_]+$") + return bool(valid_characters.match(value)) + + +def create_session_factory( + base_dir: Union[str, Path] +) -> Callable[[str], BaseChatMessageHistory]: + """Create a session ID factory that creates session IDs from a base dir. + + Args: + base_dir: Base directory to use for storing the chat histories. + + Returns: + A session ID factory that creates session IDs from a base path. + """ + base_dir_ = Path(base_dir) if isinstance(base_dir, str) else base_dir + if not base_dir_.exists(): + base_dir_.mkdir(parents=True) + + def get_chat_history(user_id: str, conversation_id: str) -> FileChatMessageHistory: + """Get a chat history from a session ID.""" + if not _is_valid_identifier(user_id): + raise ValueError( + f"User ID {user_id} is not in a valid format. " + "User ID must only contain alphanumeric characters, " + "hyphens, and underscores." + ) + if not _is_valid_identifier(conversation_id): + raise ValueError( + f"Session ID {conversation_id} is not in a valid format. " + "Session ID must only contain alphanumeric characters, " + "hyphens, and underscores." + ) + + user_dir = base_dir_ / user_id + if not user_dir.exists(): + user_dir.mkdir(parents=True) + file_path = user_dir / f"{conversation_id}.json" + return FileChatMessageHistory(file_path) + + return get_chat_history + + +app = FastAPI( + title="LangChain Server", + version="1.0", + description="Spin up a simple api server using Langchain's Runnable interfaces", +) + + +def _per_request_config_modifier( + config: Dict[str, Any], request: Request +) -> Dict[str, Any]: + """Update the config""" + config = config.copy() + configurable = config.get("configurable", {}) + # Look for a cookie named "user_id" + user_id = request.cookies.get("user_id", None) + + if user_id is None: + raise HTTPException( + status_code=400, + detail="No session ID found. Please set a cookie named 'session_id'.", + ) + + configurable["user_id"] = user_id + config["configurable"] = configurable + return config + + +# Declare a chain +prompt = ChatPromptTemplate.from_messages( + [ + ("system", "You're an assistant by the name of Bob."), + MessagesPlaceholder(variable_name="history"), + ("human", "{human_input}"), + ] +) + +chain = prompt | ChatAnthropic(model="claude-2") + + +class InputChat(TypedDict): + """Input for the chat endpoint.""" + + human_input: str + """Human input""" + + +chain_with_history = RunnableWithMessageHistory( + chain, + create_session_factory("chat_histories"), + input_messages_key="human_input", + history_messages_key="history", + session_history_config_specs=[ + ConfigurableFieldSpec( + id="user_id", + annotation=str, + name="User ID", + description="Unique identifier for the user.", + default="", + is_shared=True, + ), + ConfigurableFieldSpec( + id="conversation_id", + annotation=str, + name="Conversation ID", + description="Unique identifier for the conversation.", + # None means that the conversation ID will be generated automatically + default=None, + is_shared=True, + ), + ], +).with_types(input_type=InputChat) + + +add_routes( + app, + chain_with_history, + per_req_config_modifier=_per_request_config_modifier, + # Disable playground and batch + # 1) Playground we're passing information via headers, which is not supported via + # the playground right now. + # 2) Disable batch to avoid users being confused. Batch will work fine + # as long as users invoke it with multiple configs appropriately, but + # without validation users are likely going to forget to do that. + # In addition, there's likely little sense in support batch for a chatbot. + disabled_endpoints=["playground", "batch"], +) + +if __name__ == "__main__": + import uvicorn + + uvicorn.run(app, host="localhost", port=8000)