Spaces:
Sleeping
Sleeping
Update app.py
Browse files
app.py
CHANGED
@@ -4,11 +4,6 @@ from docx import Document
|
|
4 |
from sentence_transformers import SentenceTransformer
|
5 |
from langchain_community.vectorstores import FAISS
|
6 |
from langchain_community.embeddings import HuggingFaceEmbeddings
|
7 |
-
from langchain.docstores import InMemoryDocstore
|
8 |
-
from langchain.docstores.base import Docstore
|
9 |
-
from langchain.vectorstores.faiss import FAISS
|
10 |
-
from langchain.vectorstores.faiss import FAISSIndex
|
11 |
-
from langchain.vectorstores.faiss import IndexToDocstoreID
|
12 |
from transformers import AutoModelForSeq2SeqLM, AutoTokenizer
|
13 |
from nltk.tokenize import sent_tokenize
|
14 |
import torch
|
@@ -55,11 +50,7 @@ if os.path.exists(index_path):
|
|
55 |
faiss_index = pickle.load(f)
|
56 |
print("Loaded FAISS index from faiss_index.pkl")
|
57 |
else:
|
58 |
-
|
59 |
-
docstore = InMemoryDocstore({})
|
60 |
-
index_to_docstore_id = IndexToDocstoreID({})
|
61 |
-
faiss_index = FAISS(embedding_function=hf_embeddings, index=index, docstore=docstore, index_to_docstore_id=index_to_docstore_id)
|
62 |
-
print("Created new FAISS index")
|
63 |
|
64 |
def preprocess_text(text):
|
65 |
sentences = sent_tokenize(text)
|
@@ -81,7 +72,8 @@ def upload_files(files):
|
|
81 |
|
82 |
# Encode sentences and add to FAISS index
|
83 |
embeddings = embedding_model.encode(sentences)
|
84 |
-
|
|
|
85 |
|
86 |
# Save the updated index
|
87 |
with open(index_path, "wb") as f:
|
@@ -137,3 +129,5 @@ with gr.Blocks() as demo:
|
|
137 |
query_button.click(fn=process_and_query, inputs=[query], outputs=query_output)
|
138 |
|
139 |
demo.launch()
|
|
|
|
|
|
4 |
from sentence_transformers import SentenceTransformer
|
5 |
from langchain_community.vectorstores import FAISS
|
6 |
from langchain_community.embeddings import HuggingFaceEmbeddings
|
|
|
|
|
|
|
|
|
|
|
7 |
from transformers import AutoModelForSeq2SeqLM, AutoTokenizer
|
8 |
from nltk.tokenize import sent_tokenize
|
9 |
import torch
|
|
|
50 |
faiss_index = pickle.load(f)
|
51 |
print("Loaded FAISS index from faiss_index.pkl")
|
52 |
else:
|
53 |
+
faiss_index = FAISS(embedding_function=hf_embeddings)
|
|
|
|
|
|
|
|
|
54 |
|
55 |
def preprocess_text(text):
|
56 |
sentences = sent_tokenize(text)
|
|
|
72 |
|
73 |
# Encode sentences and add to FAISS index
|
74 |
embeddings = embedding_model.encode(sentences)
|
75 |
+
for sentence, embedding in zip(sentences, embeddings):
|
76 |
+
faiss_index.add_sentence(sentence, embedding)
|
77 |
|
78 |
# Save the updated index
|
79 |
with open(index_path, "wb") as f:
|
|
|
129 |
query_button.click(fn=process_and_query, inputs=[query], outputs=query_output)
|
130 |
|
131 |
demo.launch()
|
132 |
+
|
133 |
+
|