Update app.py

Creating embeddings & indexing for docs that we have already loaded

app.py

@@ -1,4 +1,9 @@
 from datasets import load_dataset
+from sentence_transformers import SentenceTransformer
+import faiss
+import numpy as np
+
+
 # Load a small subset (12,000 rows)
 dataset = load_dataset("wiki40b", "en", split="train[:12000]")
 
@@ -6,3 +11,17 @@ dataset = load_dataset("wiki40b", "en", split="train[:12000]")
 docs = [d["text"] for d in dataset]
 
 print("Loaded dataset with", len(docs), "documents.")
+
+# Load embedding model
+embed_model = SentenceTransformer("sentence-transformers/all-MiniLM-L6-v2")
+
+# Convert texts to embeddings
+embeddings = embed_model.encode(docs, show_progress_bar=True)
+
+# Store in FAISS index
+dimension = embeddings.shape[1]
+index = faiss.IndexFlatL2(dimension)
+index.add(np.array(embeddings))
+
+print("Stored embeddings in FAISS!")
+