Update app.py

app.py

@@ -10,6 +10,7 @@ from nltk.tokenize import sent_tokenize
 import torch
 import pickle
 import nltk
+import faiss
 
 # Ensure NLTK resources are downloaded
 try:
@@ -43,7 +44,11 @@ if os.path.exists(index_path):
         faiss_index = pickle.load(f)
         print("Loaded FAISS index from faiss_index.pkl")
 else:
-    faiss_index = FAISS()
+    # Create a new FAISS index
+    d = embedding_model.get_sentence_embedding_dimension()  # Dimension of the embeddings
+    nlist = 100  # Number of clusters (for IVF)
+    quantizer = faiss.IndexFlatL2(d)  # This is the quantizer for IVF
+    faiss_index = faiss.IndexIVFFlat(quantizer, d, nlist, faiss.METRIC_L2)
 
 def extract_text_from_pdf(pdf_path):
     text = ""
@@ -86,15 +91,15 @@ def upload_files(files):
                 sentences = preprocess_text(text)
                 embeddings = embedding_model.encode(sentences)
                 
-                for sentence, embedding in zip(sentences, embeddings):
-                    faiss_index.add_sentence(sentence, embedding)
+                for embedding in embeddings:
+                    faiss_index.add(np.array([embedding]))  # Add each embedding individually
 
             except Exception as e:
                 print(f"Error processing file '{file.name}': {e}")
                 return {"error": str(e)}
 
-        with open(index_path, "wb") as f:
-            pickle.dump(faiss_index, f)
+        # Save the updated index
+        faiss.write_index(faiss_index, index_path)
 
         return {"message": "Files processed successfully"}