Update app.py

app.py

@@ -16,11 +16,13 @@ llm_model = "TinyLlama/TinyLlama-1.1B-Chat-v1.0"
 tokenizer = AutoTokenizer.from_pretrained(llm_model)
 # pulling tokeinzer for text generation model
 
+import numpy as np
+
+datasetiter = load_dataset("Namitg02/Test", split='train', streaming=False)
+dataset = np.array(list(datasetiter))
 
-#datasetiter = load_dataset("Namitg02/Test", split='train', streaming=False)
-#dataset = list(datasetiter)
 #dataset = load_dataset("not-lain/wikipedia",revision = "embedded")
-dataset = load_dataset("epfl-llm/guidelines", split='train')
+#dataset = load_dataset("epfl-llm/guidelines", split='train')
 #Returns a list of dictionaries, each representing a row in the dataset.
 print(dataset[1])
 length = len(dataset)
@@ -35,27 +37,27 @@ embedding_model = SentenceTransformer("mixedbread-ai/mxbai-embed-large-v1")
 #doc_func = lambda x: x.text
 #dataset = list(map(doc_func, dataset))
 
-#def embedder(dataset):
-#    embeddings = embedding_model.encode(dataset["text"])
-#    dataset = dataset.add_column('embeddings', embeddings)
-#    return dataset
-#updated_dataset = dataset.map(embedder)
-#dataset['text'][:length]
+def embedder(dataset):
+    embeddings = embedding_model.encode(dataset["text"])
+    dataset = dataset.add_column('embeddings', embeddings)
+    return dataset
+updated_dataset = dataset.map(embedder)
+dataset['text'][:length]
 
 #print(embeddings)
 
-#print(updated_dataset[1])
-#print(updated_dataset[2])
-#print(dataset[1])
+print(updated_dataset[1])
+print(updated_dataset[2])
+print(dataset[1])
 
-#embedding_dim = embedding_model.get_sentence_embedding_dimension()
+embedding_dim = embedding_model.get_sentence_embedding_dimension()
 #data = FAISS.from_embeddings(embed, embedding_model)
 #data = FAISS.from_texts(docs, embedding_model)
 
 # Returns a FAISS wrapper vector store. Input is a list of strings. from_documents method used documents to Return VectorStore
 # add_embeddings
-data = dataset["clean_text"]
-#data = updated_dataset["text"]
+#data = dataset["clean_text"]
+data = updated_dataset["text"]
 
 #print(data)
 d = 384  # vectors dimension