diff --git a/examples/How_to_call_functions_for_knowledge_retrieval.ipynb b/examples/How_to_call_functions_for_knowledge_retrieval.ipynb index b438f81094..3b8e118753 100644 --- a/examples/How_to_call_functions_for_knowledge_retrieval.ipynb +++ b/examples/How_to_call_functions_for_knowledge_retrieval.ipynb @@ -125,7 +125,7 @@ "source": [ "# Set a directory to store downloaded papers\n", "data_dir = os.path.join(os.curdir, \"data\", \"papers\")\n", - "paper_dir_filepath = \"./data/arxiv_library.csv\"\n", + "paper_dir_filepath = \"./data/papers/arxiv_library.csv\"\n", "\n", "# Generate a blank dataframe where we can store downloaded files\n", "df = pd.DataFrame(list())\n", @@ -146,15 +146,14 @@ "\n", "\n", "@retry(wait=wait_random_exponential(min=1, max=40), stop=stop_after_attempt(3))\n", - "def get_articles(query, library=paper_dir_filepath, top_k=5):\n", + "def get_articles(query, library=paper_dir_filepath, top_k=10):\n", " \"\"\"This function gets the top_k articles based on a user's query, sorted by relevance.\n", " It also downloads the files and stores them in arxiv_library.csv to be retrieved by the read_article_and_summarize.\n", " \"\"\"\n", " client = arxiv.Client()\n", " search = arxiv.Search(\n", - " query = \"quantum\",\n", - " max_results = 10,\n", - " sort_by = arxiv.SortCriterion.SubmittedDate\n", + " query = query,\n", + " max_results = top_k\n", " )\n", " result_list = []\n", " for result in client.results(search):\n", @@ -192,10 +191,10 @@ { "data": { "text/plain": [ - "{'title': 'Long-range entanglement from spontaneous non-onsite symmetry breaking',\n", - " 'summary': \"We explore the states of matter arising from the spontaneous symmetry\\nbreaking (SSB) of $\\\\mathbb{Z}_2$ non-onsite symmetries. In one spatial\\ndimension, we construct a frustration-free lattice model exhibiting SSB of a\\nnon-onsite symmetry, which features the coexistence of two ground states with\\ndistinct symmetry-protected topological (SPT) orders. We analytically prove the\\ntwo-fold ground-state degeneracy and the existence of a finite energy gap.\\nFixing the symmetry sector yields a long-range entangled ground state that\\nfeatures long-range correlations among non-invertible charged operators. We\\nalso present a constant-depth measurement-feedback protocol to prepare such a\\nstate with a constant success probability in the thermodynamic limit, which may\\nbe of independent interest. Under a symmetric deformation, the SSB persists up\\nto a critical point, beyond which a gapless phase characterized by a conformal\\nfield theory emerges. In two spatial dimensions, the SSB of 1-form non-onsite\\nsymmetries leads to a long-range entangled state (SPT soup) - a condensate of\\n1d SPT along any closed loops. On a torus, there are four such locally\\nindistinguishable states that exhibit algebraic correlations between local\\noperators, which we derived via a mapping to the critical $O(2)$ loop model.\\nThis provides an intriguing example of `topological quantum criticality'. Our\\nwork reveals the exotic features of SSB of non-onsite symmetries, which may lie\\nbeyond the framework of topological holography (SymTFT).\",\n", - " 'article_url': 'http://arxiv.org/abs/2411.05004v1',\n", - " 'pdf_url': 'http://arxiv.org/pdf/2411.05004v1'}" + "{'title': 'Proximal Policy Optimization and its Dynamic Version for Sequence Generation',\n", + " 'summary': 'In sequence generation task, many works use policy gradient for model\\noptimization to tackle the intractable backpropagation issue when maximizing\\nthe non-differentiable evaluation metrics or fooling the discriminator in\\nadversarial learning. In this paper, we replace policy gradient with proximal\\npolicy optimization (PPO), which is a proved more efficient reinforcement\\nlearning algorithm, and propose a dynamic approach for PPO (PPO-dynamic). We\\ndemonstrate the efficacy of PPO and PPO-dynamic on conditional sequence\\ngeneration tasks including synthetic experiment and chit-chat chatbot. The\\nresults show that PPO and PPO-dynamic can beat policy gradient by stability and\\nperformance.',\n", + " 'article_url': 'http://arxiv.org/abs/1808.07982v1',\n", + " 'pdf_url': 'http://arxiv.org/pdf/1808.07982v1'}" ] }, "execution_count": 6, @@ -301,6 +300,8 @@ " get_articles(query)\n", " print(\"Papers downloaded, continuing\")\n", " library_df = pd.read_csv(paper_dir_filepath).reset_index()\n", + " else:\n", + " print(\"Existing papers found... Articles:\", len(library_df))\n", " library_df.columns = [\"title\", \"filepath\", \"embedding\"]\n", " library_df[\"embedding\"] = library_df[\"embedding\"].apply(ast.literal_eval)\n", " strings = strings_ranked_by_relatedness(query, library_df, top_n=1)\n", @@ -360,6 +361,7 @@ "name": "stdout", "output_type": "stream", "text": [ + "Existing papers found... Articles: 10\n", "Chunking text from paper\n", "Summarizing each chunk of text\n" ] @@ -368,7 +370,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "100%|██████████| 18/18 [00:13<00:00, 1.37it/s]\n" + "100%|███████████████████████████████████████████████████████████████████████████████████████████| 4/4 [00:05<00:00, 1.40s/it]\n" ] }, { @@ -391,29 +393,32 @@ "metadata": {}, "outputs": [ { - "name": "stdout", - "output_type": "stream", - "text": [ - "### Core Argument\n", - "- The paper presents a novel approach to quantum error correction (QEC) through the development of topological stabilizer codes based on continuous variable (CV) systems.\n", - "- It introduces a family of two-dimensional topological stabilizer codes that leverage boson condensation, generalizing existing models and addressing limitations in current QEC codes.\n", - "\n", - "### Evidence\n", - "- **Topological Stabilizer Codes**: The authors construct codes that utilize continuous variable degrees of freedom, extending models like homological rotor codes and the toric-GKP code.\n", - "- **Boson Condensation**: The codes are derived from a parent stabilizer code based on Rgauge theory, demonstrating the creation of a broad class of topological CV stabilizer codes.\n", - "- **Anyons and Theories**: The codes are characterized by anyon theories from U(1) 2n×U(1)−2m Chern-Simons theories, some of which are non-chiral and do not support gapped boundaries, indicating their unique properties.\n", - "- **Hamiltonian Analysis**: The paper examines Hamiltonians associated with these codes, revealing that they can transition from a gapless to a gapped spectrum through quadratic perturbations.\n", - "- **Scalability**: The research emphasizes the potential for scalable QEC codes that are intrinsic to CV systems, which is crucial for reducing logical error rates in complex quantum computations.\n", - "\n", - "### Conclusions\n", - "- The introduction of topological stabilizer codes based on continuous variables represents a significant advancement in the field of quantum error correction, providing a framework for developing fault-tolerant codes that can protect against logical errors.\n", - "- The findings suggest that these codes may not be realizable through traditional concatenation methods, indicating the potential for new classes of QEC codes that are fundamentally different from existing finite-dimensional systems.\n", - "- Future research directions include exploring many-body generalizations of quantum codes and the practical implementation of these theoretical models in experimental platforms, such as superconducting qubits.\n" - ] + "data": { + "text/markdown": [ + "### Core Argument\n", + "- The paper argues that Proximal Policy Optimization (PPO) and its dynamic variant (PPO-dynamic) significantly improve sequence generation tasks, particularly for chit-chat chatbots, by addressing the instability and suboptimal performance associated with traditional policy gradient methods.\n", + "\n", + "### Evidence\n", + "- **Challenges with Traditional Methods**: Traditional policy gradient methods, like REINFORCE, suffer from unstable training and poor performance due to large updates and similar action tendencies, especially in non-differentiable evaluation contexts (e.g., BLEU scores).\n", + "- **PPO Advantages**: PPO regularizes policy updates, enhancing training stability and enabling the generation of coherent and diverse chatbot responses.\n", + "- **Dynamic PPO Approach**: PPO-dynamic introduces adaptive constraints on KL-divergence, allowing for dynamic adjustments based on action probabilities, which leads to improved training performance.\n", + "- **Experimental Validation**: The authors conducted experiments on synthetic counting tasks and real-world chit-chat scenarios, demonstrating that PPO and PPO-dynamic outperform traditional methods like REINFORCE and SeqGAN in terms of stability and performance metrics (e.g., BLEU-2 scores).\n", + "- **Results**: PPO-dynamic showed faster convergence and higher precision in the counting task, and it achieved the best performance in the chit-chat task, indicating its effectiveness in generating diverse and contextually appropriate responses.\n", + "\n", + "### Conclusions\n", + "- The introduction of PPO and PPO-dynamic enhances the training stability and output diversity in sequence generation tasks, making them more suitable for applications like chatbots.\n", + "- The dynamic variant of PPO not only improves performance but also accelerates convergence, addressing the limitations of traditional policy gradient methods and providing a robust framework for reinforcement learning in sequence generation." + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" } ], "source": [ - "print(chat_test_response.choices[0].message.content)\n" + "display(Markdown(chat_test_response.choices[0].message.content))\n" ] }, { @@ -633,19 +638,49 @@ { "data": { "text/markdown": [ - "It looks like none of the listed papers directly address Proximal Policy Optimization (PPO) in reinforcement learning. However, if you are looking for a summary of how PPO works, I can provide that information based on general knowledge.\n", + "Here are some recent papers that discuss Proximal Policy Optimization (PPO) in reinforcement learning, explaining its mechanics and various enhancements:\n", + "\n", + "1. **[Proximal Policy Optimization and its Dynamic Version for Sequence Generation](http://arxiv.org/abs/1808.07982v1)** \n", + " - *Summary:* This paper applies PPO to sequence generation tasks, demonstrating that it outperforms traditional policy gradient methods in terms of stability and performance. It introduces a dynamic version of PPO for these tasks.\n", + " - [PDF](http://arxiv.org/pdf/1808.07982v1)\n", + "\n", + "2. **[CIM-PPO: Proximal Policy Optimization with Liu-Correntropy Induced Metric](http://arxiv.org/abs/2110.10522v3)** \n", + " - *Summary:* This work investigates the asymmetry in KL divergence in PPO-KL and proposes PPO-CIM as an enhanced version with lower computation costs and improved policy updates, validated through experiments on continuous-action tasks.\n", + " - [PDF](http://arxiv.org/pdf/2110.10522v3)\n", + "\n", + "3. **[A2C is a special case of PPO](http://arxiv.org/abs/2205.09123v1)** \n", + " - *Summary:* This paper shows that A2C can be viewed as a special case of PPO, providing theoretical justifications and empirical evidence demonstrating their equivalence under controlled conditions.\n", + " - [PDF](http://arxiv.org/pdf/2205.09123v1)\n", "\n", - "PPO is a popular reinforcement learning algorithm that seeks to optimize an agent's policy in a way that is both effective and stable. The key ideas behind PPO include:\n", + "4. **[Proximal Policy Optimization via Enhanced Exploration Efficiency](http://arxiv.org/abs/2011.05525v1)** \n", + " - *Summary:* This paper enhances the PPO algorithm by improving exploration strategies, proposing IEM-PPO, which shows better sample efficiency and rewards than standard methods in complex environments.\n", + " - [PDF](http://arxiv.org/pdf/2011.05525v1)\n", "\n", - "1. **Clipping Objective**: PPO modifies the objective function using a clipped surrogate objective. This avoids excessively large updates, which can destabilize training. The clipped objective restricts how much the probability ratio between the new and old policies can differ, allowing for better control over policy updates.\n", + "5. **[ReMax: A Simple, Effective, and Efficient Reinforcement Learning Method for Aligning Large Language Models](http://arxiv.org/abs/2310.10505v4)** \n", + " - *Summary:* The ReMax method is proposed as an alternative to PPO for training large language models, reducing hyper-parameter tuning complexities and enhancing training efficiency.\n", + " - [PDF](http://arxiv.org/pdf/2310.10505v4)\n", "\n", - "2. **Policy Gradient**: PPO uses policy gradient methods, where the focus is on directly adjusting the policy based on the agent's performance after taking actions in the environment.\n", + "6. **[Reward Scale Robustness for Proximal Policy Optimization via DreamerV3 Tricks](http://arxiv.org/abs/2310.17805v1)** \n", + " - *Summary:* This work examines the applicability of DreamerV3's tricks to PPO, revealing mixed outcomes and providing insights into the clipping mechanism in PPO's performance.\n", + " - [PDF](http://arxiv.org/pdf/2310.17805v1)\n", "\n", - "3. **Multiple Epochs**: PPO often utilizes multiple epochs of training on the same batch of data, which improves sample efficiency as it allows the agent to learn from the same data multiple times.\n", + "7. **[Neural PPO-Clip Attains Global Optimality: A Hinge Loss Perspective](http://arxiv.org/abs/2110.13799v4)** \n", + " - *Summary:* This paper establishes a theoretical grounding for PPO-Clip and introduces new interpretive frameworks for its mechanics, showing improved convergence properties.\n", + " - [PDF](http://arxiv.org/pdf/2110.13799v4)\n", "\n", - "4. **Generalized Advantage Estimation (GAE)**: To reduce the variance of the policy gradient estimates, PPO frequently employs GAE, which provides a more stable estimate of returns.\n", + "8. **[Colored Noise in PPO: Improved Exploration and Performance through Correlated Action Sampling](http://dx.doi.org/10.1609/aaai.v38i11.29139)** \n", + " - *Summary:* This study proposes a variant of PPO using correlated noise for improved exploration, demonstrating enhanced performance over traditional approaches.\n", + " - [PDF](http://arxiv.org/abs/2312.11091v2)\n", "\n", - "If you have specific questions about reinforcement learning or want references to further reading, feel free to ask!" + "9. **[A dynamical clipping approach with task feedback for Proximal Policy Optimization](http://arxiv.org/abs/2312.07624v3)** \n", + " - *Summary:* The paper presents Pb-PPO, which dynamically adjusts the clipping bounds in PPO to enhance returns, showing improved performance across various tasks.\n", + " - [PDF](http://arxiv.org/pdf/2312.07624v3)\n", + "\n", + "10. **[PPO-UE: Proximal Policy Optimization via Uncertainty-Aware Exploration](http://arxiv.org/abs/2212.06343v1)** \n", + " - *Summary:* Introducing PPO-UE, which incorporates uncertainty-aware exploration, this paper shows improvements in convergence speed and performance compared to standard PPO.\n", + " - [PDF](http://arxiv.org/pdf/2212.06343v1)\n", + "\n", + "These papers provide a comprehensive view of the developments and enhancements in PPO and how it operates within the reinforcement learning framework. You can click on the titles to access the full articles." ], "text/plain": [ "" @@ -677,18 +712,44 @@ "output_type": "stream", "text": [ "Function generation requested, calling function\n", - "Getting search results\n", - "Got search results, summarizing content\n" + "Finding and reading paper\n", + "Existing papers found... Articles: 20\n", + "Chunking text from paper\n", + "Summarizing each chunk of text\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|███████████████████████████████████████████████████████████████████████████████████████████| 4/4 [00:04<00:00, 1.21s/it]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Summarizing into overall summary\n" ] }, { "data": { "text/markdown": [ - "It seems that I couldn't find any specific paper that discusses \"PPO sequence generation.\" If you meant a particular paper on PPO in reinforcement learning, please provide its title or broad topic. Alternatively, if you want to know about PPO in the context of generating sequences or trajectories, I can provide a detailed explanation based on existing knowledge in reinforcement learning.\n", + "### Core Argument\n", + "- The paper argues for the adoption of Proximal Policy Optimization (PPO) and its dynamic variant (PPO-dynamic) as superior methods for sequence generation tasks, particularly in the context of chit-chat chatbots, compared to traditional policy gradient methods.\n", + "- It highlights the instability and suboptimal performance of traditional policy gradient methods, such as REINFORCE, and presents PPO as a more stable and efficient alternative.\n", "\n", - "If you are referring to a specific recent study, I recommend checking relevant platforms like arXiv or Google Scholar for the latest publications on PPO and sequence generation.\n", + "### Evidence\n", + "- **Challenges with Policy Gradient**: Traditional methods lead to unstable training and poor performance due to large updates and similar action tendencies, especially in non-differentiable evaluation metrics like BLEU scores.\n", + "- **PPO Advantages**: PPO regularizes policy updates, enhancing stability and coherence in chatbot responses.\n", + "- **Dynamic PPO Approach**: PPO-dynamic introduces dynamic adjustments to the KL-divergence bounds, allowing for more flexible and effective training.\n", + "- **Experimental Validation**: Experiments on synthetic tasks and real-world chit-chat scenarios demonstrate that PPO and PPO-dynamic outperform REINFORCE and other algorithms (like MIXER and SeqGAN) in terms of stability and performance metrics, including BLEU-2 scores.\n", + "- **Results**: PPO-dynamic showed significant improvements in precision on counting tasks and achieved the highest BLEU-2 score for chatbot responses, indicating better performance in generating diverse and accurate outputs.\n", "\n", - "Let me know how you'd like to proceed!" + "### Conclusions\n", + "- The paper concludes that replacing traditional policy gradient methods with PPO, particularly the dynamic version, leads to more stable training and faster convergence in sequence generation tasks.\n", + "- The proposed PPO-dynamic method enhances the training process by dynamically adjusting constraints, resulting in improved performance and efficiency in generating human-like conversational agents.\n", + "- Future research directions are suggested to further explore the potential of PPO and its adaptations in natural language processing applications." ], "text/plain": [ "" @@ -713,7 +774,7 @@ ], "metadata": { "kernelspec": { - "display_name": "openai-cookbook", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, @@ -727,7 +788,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.7" + "version": "3.11.8" } }, "nbformat": 4, diff --git a/examples/vector_databases/qdrant/Getting_started_with_Qdrant_and_OpenAI.ipynb b/examples/vector_databases/qdrant/Getting_started_with_Qdrant_and_OpenAI.ipynb index 63fa7df9cd..6764d5f677 100644 --- a/examples/vector_databases/qdrant/Getting_started_with_Qdrant_and_OpenAI.ipynb +++ b/examples/vector_databases/qdrant/Getting_started_with_Qdrant_and_OpenAI.ipynb @@ -32,7 +32,7 @@ ] }, { - "cell_type": "markdown", + "cell_type": "raw", "metadata": {}, "source": [ "## Prerequisites\n", @@ -41,7 +41,7 @@ "\n", "1. Qdrant server instance. In our case a local Docker container.\n", "2. The [qdrant-client](https://github.com/qdrant/qdrant_client) library to interact with the vector database.\n", - "3. An [OpenAI API key](https://beta.openai.com/account/api-keys).\n", + "3. An [OpenAI API key](https://platform.openai.com/settings/organization/api-keys).\n", "\n", "### Start Qdrant server\n", "\n", @@ -62,12 +62,14 @@ "name": "stdout", "output_type": "stream", "text": [ - "qdrant_qdrant_1 is up-to-date\r\n" + "\u001b[1A\u001b[1B\u001b[0G\u001b[?25l[+] Running 1/0\n", + " \u001b[32m✔\u001b[0m Container qdrant-qdrant-1 \u001b[32mRunning\u001b[0m \u001b[34m0.0s \u001b[0m\n", + "\u001b[?25h" ] } ], "source": [ - "! docker-compose up -d" + "! docker compose up -d" ] }, { @@ -91,7 +93,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "{\"title\":\"qdrant - vector search engine\",\"version\":\"1.0.1\"}" + "{\"title\":\"qdrant - vector search engine\",\"version\":\"1.3.0\"}" ] } ], @@ -137,7 +139,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 4, "metadata": {}, "outputs": [], "source": [ @@ -146,7 +148,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 5, "metadata": { "ExecuteTime": { "end_time": "2023-02-16T12:05:05.730338Z", @@ -187,7 +189,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 6, "metadata": { "ExecuteTime": { "end_time": "2023-02-16T12:05:06.827143Z", @@ -213,7 +215,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 7, "metadata": { "ExecuteTime": { "end_time": "2023-02-16T12:05:06.848488Z", @@ -227,13 +229,13 @@ "CollectionsResponse(collections=[])" ] }, - "execution_count": 6, + "execution_count": 7, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "client.get_collections()" + "client.get_collections()\n" ] }, { @@ -247,14 +249,32 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 8, "metadata": { "ExecuteTime": { "end_time": "2023-02-16T12:05:37.371951Z", "start_time": "2023-02-16T12:05:06.851634Z" } }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "100% [......................................................................] 698933052 / 698933052" + ] + }, + { + "data": { + "text/plain": [ + "'vector_database_wikipedia_articles_embedded (9).zip'" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "import wget\n", "\n", @@ -273,7 +293,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 9, "metadata": { "ExecuteTime": { "end_time": "2023-02-16T12:06:01.538851Z", @@ -297,7 +317,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 10, "metadata": { "ExecuteTime": { "end_time": "2023-02-16T12:17:35.483972Z", @@ -420,7 +440,7 @@ "4 [0.021524671465158463, 0.018522677943110466, -... 4 " ] }, - "execution_count": 9, + "execution_count": 10, "metadata": {}, "output_type": "execute_result" } @@ -450,7 +470,7 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 11, "metadata": { "ExecuteTime": { "end_time": "2023-02-16T12:17:36.366066Z", @@ -464,7 +484,7 @@ "True" ] }, - "execution_count": 10, + "execution_count": 11, "metadata": {}, "output_type": "execute_result" } @@ -474,7 +494,7 @@ "\n", "vector_size = len(article_df[\"content_vector\"][0])\n", "\n", - "client.recreate_collection(\n", + "client.create_collection(\n", " collection_name=\"Articles\",\n", " vectors_config={\n", " \"title\": rest.VectorParams(\n", @@ -491,7 +511,7 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 12, "metadata": { "ExecuteTime": { "end_time": "2023-02-16T12:30:37.518210Z", @@ -505,7 +525,7 @@ "UpdateResult(operation_id=0, status=)" ] }, - "execution_count": 11, + "execution_count": 12, "metadata": {}, "output_type": "execute_result" } @@ -529,7 +549,7 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 13, "metadata": { "ExecuteTime": { "end_time": "2023-02-16T12:30:40.675202Z", @@ -543,7 +563,7 @@ "CountResult(count=25000)" ] }, - "execution_count": 16, + "execution_count": 13, "metadata": {}, "output_type": "execute_result" } @@ -559,12 +579,12 @@ "source": [ "## Search data\n", "\n", - "Once the data is put into Qdrant we will start querying the collection for the closest vectors. We may provide an additional parameter `vector_name` to switch from title to content based search. Since the precomputed embeddings were created with `text-embedding-3-small` OpenAI model we also have to use it during search.\n" + "Once the data is put into Qdrant we will start querying the collection for the closest vectors. We may provide an additional parameter `vector_name` to switch from title to content based search. Since the precomputed embeddings were created with `text-embedding-ada-002` OpenAI model we also have to use it during search.\n" ] }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 14, "metadata": { "ExecuteTime": { "end_time": "2023-02-16T12:30:38.024370Z", @@ -573,14 +593,16 @@ }, "outputs": [], "source": [ - "import openai\n", + "from openai import OpenAI\n", + "\n", + "openai_client = OpenAI()\n", "\n", "def query_qdrant(query, collection_name, vector_name=\"title\", top_k=20):\n", " # Creates embedding vector from user query\n", - " embedded_query = openai.Embedding.create(\n", + " embedded_query = openai_client.embeddings.create(\n", " input=query,\n", - " model=\"text-embedding-3-small\",\n", - " )[\"data\"][0][\"embedding\"]\n", + " model=\"text-embedding-ada-002\",\n", + " ).data[0].embedding\n", "\n", " query_results = client.search(\n", " collection_name=collection_name,\n", @@ -595,7 +617,7 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 15, "metadata": { "ExecuteTime": { "end_time": "2023-02-16T12:30:39.379566Z", @@ -608,7 +630,7 @@ "output_type": "stream", "text": [ "1. Museum of Modern Art (Score: 0.875)\n", - "2. Western Europe (Score: 0.868)\n", + "2. Western Europe (Score: 0.867)\n", "3. Renaissance art (Score: 0.864)\n", "4. Pop art (Score: 0.86)\n", "5. Northern Europe (Score: 0.855)\n", @@ -621,8 +643,8 @@ "12. Byzantine art (Score: 0.841)\n", "13. Postmodernism (Score: 0.84)\n", "14. Eastern Europe (Score: 0.839)\n", - "15. Europe (Score: 0.839)\n", - "16. Cubism (Score: 0.839)\n", + "15. Cubism (Score: 0.839)\n", + "16. Europe (Score: 0.839)\n", "17. Impressionism (Score: 0.838)\n", "18. Bauhaus (Score: 0.838)\n", "19. Surrealism (Score: 0.837)\n", @@ -638,7 +660,7 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 16, "metadata": { "ExecuteTime": { "end_time": "2023-02-16T12:30:40.652676Z", @@ -652,7 +674,7 @@ "text": [ "1. Battle of Bannockburn (Score: 0.869)\n", "2. Wars of Scottish Independence (Score: 0.861)\n", - "3. 1651 (Score: 0.853)\n", + "3. 1651 (Score: 0.852)\n", "4. First War of Scottish Independence (Score: 0.85)\n", "5. Robert I of Scotland (Score: 0.846)\n", "6. 841 (Score: 0.844)\n", @@ -665,11 +687,11 @@ "13. 1746 (Score: 0.83)\n", "14. 1040s (Score: 0.828)\n", "15. 1106 (Score: 0.827)\n", - "16. 1304 (Score: 0.827)\n", + "16. 1304 (Score: 0.826)\n", "17. David II of Scotland (Score: 0.825)\n", "18. Braveheart (Score: 0.824)\n", "19. 1124 (Score: 0.824)\n", - "20. July 27 (Score: 0.823)\n" + "20. Second War of Scottish Independence (Score: 0.823)\n" ] } ], @@ -679,13 +701,6 @@ "for i, article in enumerate(query_results):\n", " print(f\"{i + 1}. {article.payload['title']} (Score: {round(article.score, 3)})\")" ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] } ], "metadata": { @@ -704,9 +719,9 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.6" + "version": "3.11.8" } }, "nbformat": 4, - "nbformat_minor": 1 + "nbformat_minor": 4 } diff --git a/examples/vector_databases/qdrant/Using_Qdrant_for_embeddings_search.ipynb b/examples/vector_databases/qdrant/Using_Qdrant_for_embeddings_search.ipynb index 188b487ffd..ffb598d24a 100644 --- a/examples/vector_databases/qdrant/Using_Qdrant_for_embeddings_search.ipynb +++ b/examples/vector_databases/qdrant/Using_Qdrant_for_embeddings_search.ipynb @@ -53,7 +53,7 @@ }, { "cell_type": "code", - "execution_count": 23, + "execution_count": 2, "id": "5be94df6", "metadata": { "ExecuteTime": { @@ -90,7 +90,7 @@ }, { "cell_type": "code", - "execution_count": 24, + "execution_count": 3, "id": "5dff8b55", "metadata": { "ExecuteTime": { @@ -98,21 +98,37 @@ "start_time": "2024-05-21T23:49:41.132888Z" } }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "100% [......................................................................] 698933052 / 698933052" + ] + }, + { + "data": { + "text/plain": [ + "'vector_database_wikipedia_articles_embedded (10).zip'" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ - "import requests\n", + "import wget\n", "\n", - "embeddings_url = 'https://cdn.openai.com/API/examples/data/vector_database_wikipedia_articles_embedded.zip'\n", + "embeddings_url = \"https://cdn.openai.com/API/examples/data/vector_database_wikipedia_articles_embedded.zip\"\n", "\n", "# The file is ~700 MB so this will take some time\n", - "response = requests.get(embeddings_url, verify=True) # Set verify=False to bypass SSL verification\n", - "with open('vector_database_wikipedia_articles_embedded.zip', 'wb') as file:\n", - " file.write(response.content)" + "wget.download(embeddings_url)" ] }, { "cell_type": "code", - "execution_count": 25, + "execution_count": 4, "id": "21097972", "metadata": { "ExecuteTime": { @@ -129,7 +145,7 @@ }, { "cell_type": "code", - "execution_count": 26, + "execution_count": 5, "id": "70bbd8ba", "metadata": { "ExecuteTime": { @@ -144,7 +160,7 @@ }, { "cell_type": "code", - "execution_count": 27, + "execution_count": 6, "id": "1721e45d", "metadata": { "ExecuteTime": { @@ -268,7 +284,7 @@ "4 [0.021524671465158463, 0.018522677943110466, -... 4 " ] }, - "execution_count": 27, + "execution_count": 6, "metadata": {}, "output_type": "execute_result" } @@ -279,7 +295,7 @@ }, { "cell_type": "code", - "execution_count": 28, + "execution_count": 7, "id": "960b82af", "metadata": { "ExecuteTime": { @@ -299,7 +315,7 @@ }, { "cell_type": "code", - "execution_count": 29, + "execution_count": 8, "id": "a334ab8b", "metadata": { "ExecuteTime": { @@ -364,7 +380,27 @@ }, { "cell_type": "code", - "execution_count": 30, + "execution_count": 9, + "id": "fc9a0203-30c2-4a7c-95cb-c5bacc908a4d", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[1A\u001b[1B\u001b[0G\u001b[?25l[+] Running 1/0\n", + " \u001b[32m✔\u001b[0m Container qdrant-qdrant-1 \u001b[32mRunning\u001b[0m \u001b[34m0.0s \u001b[0m\n", + "\u001b[?25h" + ] + } + ], + "source": [ + "! docker compose up -d" + ] + }, + { + "cell_type": "code", + "execution_count": 10, "id": "76d697e9", "metadata": { "ExecuteTime": { @@ -379,7 +415,7 @@ }, { "cell_type": "code", - "execution_count": 31, + "execution_count": 11, "id": "1deeb539", "metadata": { "ExecuteTime": { @@ -394,7 +430,7 @@ "CollectionsResponse(collections=[CollectionDescription(name='Articles')])" ] }, - "execution_count": 31, + "execution_count": 11, "metadata": {}, "output_type": "execute_result" } @@ -417,7 +453,7 @@ }, { "cell_type": "code", - "execution_count": 32, + "execution_count": 12, "id": "1a84ee1d", "metadata": { "ExecuteTime": { @@ -432,7 +468,7 @@ }, { "cell_type": "code", - "execution_count": 33, + "execution_count": 13, "id": "00876f92", "metadata": { "ExecuteTime": { @@ -447,7 +483,7 @@ "True" ] }, - "execution_count": 33, + "execution_count": 13, "metadata": {}, "output_type": "execute_result" } @@ -474,7 +510,7 @@ }, { "cell_type": "code", - "execution_count": 34, + "execution_count": 14, "id": "9f39a8c395554ca3", "metadata": { "ExecuteTime": { @@ -489,7 +525,7 @@ "True" ] }, - "execution_count": 34, + "execution_count": 14, "metadata": {}, "output_type": "execute_result" } @@ -522,7 +558,7 @@ }, { "cell_type": "code", - "execution_count": 35, + "execution_count": 15, "id": "f24e76ab", "metadata": { "ExecuteTime": { @@ -535,7 +571,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "Upserting articles: 100%|██████████| 25000/25000 [01:18<00:00, 318.84it/s]\n" + "Upserting articles: 100%|█████████████████████████████████████████████████████████████████████████████████████| 25000/25000 [02:52<00:00, 144.82it/s]\n" ] } ], @@ -568,7 +604,7 @@ }, { "cell_type": "code", - "execution_count": 36, + "execution_count": 16, "id": "d1188a12", "metadata": { "ExecuteTime": { @@ -583,7 +619,7 @@ "CountResult(count=25000)" ] }, - "execution_count": 36, + "execution_count": 16, "metadata": {}, "output_type": "execute_result" } @@ -600,12 +636,12 @@ "source": [ "### Search Data\n", "\n", - "Once the data is put into Qdrant we will start querying the collection for the closest vectors. We may provide an additional parameter `vector_name` to switch from title to content based search." + "Once the data is put into Qdrant we will start querying the collection for the closest vectors. We may provide an additional parameter `vector_name` to switch from title to content based search. Ensure you use the text-embedding-ada-002 model as the original embeddings in file were created with this model." ] }, { "cell_type": "code", - "execution_count": 37, + "execution_count": 17, "id": "f1bac4ef", "metadata": { "ExecuteTime": { @@ -637,7 +673,7 @@ }, { "cell_type": "code", - "execution_count": 39, + "execution_count": 18, "id": "aa92f3d3", "metadata": { "ExecuteTime": { @@ -681,7 +717,7 @@ }, { "cell_type": "code", - "execution_count": 40, + "execution_count": 19, "id": "7ed116b8", "metadata": { "ExecuteTime": { @@ -696,7 +732,7 @@ "text": [ "1. Battle of Bannockburn, URL: https://simple.wikipedia.org/wiki/Battle%20of%20Bannockburn (Score: 0.869)\n", "2. Wars of Scottish Independence, URL: https://simple.wikipedia.org/wiki/Wars%20of%20Scottish%20Independence (Score: 0.861)\n", - "3. 1651, URL: https://simple.wikipedia.org/wiki/1651 (Score: 0.853)\n", + "3. 1651, URL: https://simple.wikipedia.org/wiki/1651 (Score: 0.852)\n", "4. First War of Scottish Independence, URL: https://simple.wikipedia.org/wiki/First%20War%20of%20Scottish%20Independence (Score: 0.85)\n", "5. Robert I of Scotland, URL: https://simple.wikipedia.org/wiki/Robert%20I%20of%20Scotland (Score: 0.846)\n", "6. 841, URL: https://simple.wikipedia.org/wiki/841 (Score: 0.844)\n", @@ -723,14 +759,6 @@ "for i, article in enumerate(query_results):\n", " print(f'{i + 1}. {article.payload[\"title\"]}, URL: {article.payload[\"url\"]} (Score: {round(article.score, 3)})')" ] - }, - { - "cell_type": "code", - "execution_count": 19, - "id": "cd4f750dc6daa2e8", - "metadata": {}, - "outputs": [], - "source": [] } ], "metadata": { @@ -749,7 +777,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.11" + "version": "3.11.8" }, "vscode": { "interpreter": {