fixing app.py

app.py

@@ -61,16 +61,16 @@ def main():
             " ",
             "",]
         st.session_state.docs_processed = split_documents(
-            512,  # We choose a chunk size adapted to our model
+            400,  # We choose a chunk size adapted to our model
             st.session_state.raw_document_base,
             #tokenizer_name=EMBEDDING_MODEL_NAME,
             separator=st.session_state.MARKDOWN_SEPARATORS
         )
-        st.session_state.embedding_model=NVIDIAEmbeddings()
+        st.session_state.embedding_model=NVIDIAEmbeddings(model="NV-Embed-QA", truncate="END")
         st.session_state.KNOWLEDGE_VECTOR_DATABASE= init_vectorDB_from_doc(st.session_state.docs_processed, 
                                                                            st.session_state.embedding_model)
     if (user_query) and (st.button("Get Answer")):
-        num_doc_before_rerank=15 
+        num_doc_before_rerank=5 
         st.session_state.retriever= st.session_state.KNOWLEDGE_VECTOR_DATABASE.as_retriever(search_type="similarity",
                                                                                             search_kwargs={"k": num_doc_before_rerank})
         

src/data_preparation.py

@@ -12,31 +12,35 @@ from typing import List, Optional
 #from langchain import HuggingFacePipeline
 #from langchain.chains import RetrievalQA
 
-EMBEDDING_MODEL_NAME = "OrdalieTech/Solon-embeddings-large-0.1"
+#EMBEDDING_MODEL_NAME = "OrdalieTech/Solon-embeddings-large-0.1"
 
 
 def split_documents(
     chunk_size: int,
     knowledge_base: List[LangchainDocument],
-    tokenizer_name: Optional[str] = EMBEDDING_MODEL_NAME,
+    #tokenizer_name: Optional[str] = EMBEDDING_MODEL_NAME,
     separator:List[str]=None,
 ) -> List[LangchainDocument]:
     """
     Split documents into chunks of maximum size `chunk_size` tokens and return a list of documents.
     """
-    text_splitter = RecursiveCharacterTextSplitter.from_huggingface_tokenizer(
-        AutoTokenizer.from_pretrained(tokenizer_name),
-        chunk_size=chunk_size,
-        chunk_overlap=int(chunk_size / 10),
-        add_start_index=True,
-        strip_whitespace=True,
-        separators=separator,
-    )
+    #text_splitter = RecursiveCharacterTextSplitter.from_huggingface_tokenizer(
+    #    AutoTokenizer.from_pretrained(tokenizer_name),
+    #    chunk_size=chunk_size,
+    #    chunk_overlap=int(chunk_size / 10),
+    #    add_start_index=True,
+    #    strip_whitespace=True,
+    #    separators=separator,
+    #)
+    text_splitter= RecursiveCharacterTextSplitter( chunk_size=chunk_size,
+                                                  chunk_overlap=int(chunk_size / 10),
+                                                  strip_whitespace=True,
+                                                  separators=separator)
 
     docs_processed = []
-    for doc in knowledge_base:
-        docs_processed += text_splitter.split_documents([doc])
-
+    #for doc in knowledge_base:
+    #    docs_processed += text_splitter.split_documents([doc])
+    docs_processed=text_splitter.split_documents(knowledge_base)
     # Remove duplicates
     unique_texts = {}
     docs_processed_unique = []