Update app.py

app.py

@@ -61,7 +61,7 @@ def indian_to_english(sentence):
 
 
 llm_model = "mistralai/Mixtral-8x7B-Instruct-v0.1"
-
+tokenizer_name = "thenlper/gte-small"
 
 # default_persist_directory = './chroma_HF/'
 list_llm = ["mistralai/Mistral-7B-Instruct-v0.2", "mistralai/Mixtral-8x7B-Instruct-v0.1", "mistralai/Mistral-7B-Instruct-v0.1", \
@@ -82,11 +82,14 @@ def load_doc(list_file_path, chunk_size, chunk_overlap):
     for loader in loaders:
         pages.extend(loader.load())
     # text_splitter = RecursiveCharacterTextSplitter(chunk_size = 600, chunk_overlap = 50)
-    text_splitter = RecursiveCharacterTextSplitter(
-        chunk_size = chunk_size, 
-        chunk_overlap = chunk_overlap)
+    text_splitter = RecursiveCharacterTextSplitter.from_huggingface_tokenizer(
+        AutoTokenizer.from_pretrained(tokenizer_name),
+        chunk_size=chunk_size,
+        chunk_overlap=chunk_overlap,
+        strip_whitespace=True)
     doc_splits = text_splitter.split_documents(pages)
     return doc_splits
+    
 
 
 # Create vector database
@@ -247,9 +250,9 @@ def demo():
             db_btn = gr.Radio(["ChromaDB"], label="Vector database type", value = "ChromaDB", type="index", info="Choose your vector database",visible=False)
         with gr.Accordion("Advanced options - Document text splitter", open=False, visible=False):
             with gr.Row():
-                slider_chunk_size = gr.Slider(value=20000, label="Chunk size", info="Chunk size", interactive=False, visible=False)
+                slider_chunk_size = gr.Slider(value=512, label="Chunk size", info="Chunk size", interactive=False, visible=False)
             with gr.Row():
-                slider_chunk_overlap = gr.Slider(value=2000, label="Chunk overlap", info="Chunk overlap", interactive=False, visible=False)
+                slider_chunk_overlap = gr.Slider(value=128, label="Chunk overlap", info="Chunk overlap", interactive=False, visible=False)
 
         with gr.Accordion("Advanced options - LLM model", open=False, visible=False):
             with gr.Row():