drop short docs before ingestion (#268)

langchain-ai · Feb 13, 2024 · bbe56e9 · bbe56e9
1 parent 7356d00
commit bbe56e9
Showing 1 changed file with 1 addition and 0 deletions.
diff --git a/ingest.py b/ingest.py
@@ -130,6 +130,7 @@ def ingest_docs():
     docs_transformed = text_splitter.split_documents(
         docs_from_documentation + docs_from_api + docs_from_langsmith
     )
+    docs_transformed = [doc for doc in docs_transformed if len(doc.page_content) > 10]
 
     # We try to return 'source' and 'title' metadata when querying vector store and
     # Weaviate will error at query time if one of the attributes is missing from a