Update app.py

app.py

@@ -48,7 +48,7 @@ embeddings = HuggingFaceEmbeddings(
     encode_kwargs=encode_kwargs
 )
 
-chunk_size = 500
+
 
 # creating a pdf reader object
 
@@ -60,10 +60,29 @@ def get_text_embedding(text):
 
     return embeddings.embed_query(text)
 
+
+# FAISS index
 doc_path = hf_hub_download(repo_id="xavierbarbier/rag_ngap", filename="resource/embeddings_ngap.faiss", repo_type="space")
 
 index = faiss.read_index(doc_path)
 
+# Chunks
+doc_path = hf_hub_download(repo_id="xavierbarbier/rag_ngap", filename="resource/NGAP 01042024.pdf", repo_type="space")
+
+reader = PdfReader(doc_path)
+
+text = []
+for p in np.arange(0, len(reader.pages), 1):
+  page = reader.pages[int(p)]
+
+  # extracting text from page
+  text.append(page.extract_text())
+
+text = ' '.join(text)
+
+chunk_size = 2048
+chunks = [text[i:i + chunk_size] for i in range(0, len(text), chunk_size)]
+
 def qa(question):