Update app.py

app.py

@@ -73,6 +73,10 @@ else:
         pickle.dump(index, f)
         print("Created new FAISS index and saved to faiss_index.pkl")
 
+def preprocess_text(text):
+    # Add more preprocessing steps if necessary
+    return text.strip()
+
 def upload_files(files):
     global index, document_texts
     try:
@@ -89,6 +93,7 @@ def upload_files(files):
 
             # Process the text and update FAISS index
             sentences = text.split("\n")
+            sentences = [preprocess_text(sentence) for sentence in sentences if sentence.strip()]
             embeddings = embedding_model.encode(sentences)
             print(f"Embeddings shape: {embeddings.shape}")  # Debug: Show the shape of the embeddings
             index.add(np.array(embeddings))
@@ -125,6 +130,10 @@ def query_text(text):
                 top_documents.append(document_texts[idx])  # Append the actual sentences for the response
             else:
                 print(f"Invalid index found: {idx}")
+        
+        # Remove duplicates
+        top_documents = list(dict.fromkeys(top_documents))
+        
         return top_documents
     except Exception as e:
         print(f"Error querying text: {e}")
@@ -161,6 +170,7 @@ demo.launch()
 
 
 
+