moved to recursivechar

train/faq.py

@@ -1,7 +1,7 @@
 
 def train():
     from langchain_community.document_loaders.csv_loader import CSVLoader
-    from langchain.text_splitter import CharacterTextSplitter
+    from langchain.text_splitter import RecursiveCharacterTextSplitter
     from langchain_openai import OpenAIEmbeddings
     from langchain_community.vectorstores.faiss import FAISS
     from dotenv import load_dotenv
@@ -10,7 +10,7 @@ def train():
     documents = WebBaseLoader("https://rise.mmu.ac.uk/what-is-rise/").load()
 
     # Split document in chunks
-    text_splitter = CharacterTextSplitter(chunk_size=1000, chunk_overlap=30)
+    text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=150)
     docs = text_splitter.split_documents(documents=documents)
 
     embeddings = OpenAIEmbeddings()