Spaces:
Runtime error
Runtime error
Update app.py
Browse files
app.py
CHANGED
@@ -4,6 +4,11 @@ from docx import Document
|
|
4 |
from sentence_transformers import SentenceTransformer
|
5 |
from langchain_community.vectorstores import FAISS
|
6 |
from langchain_community.embeddings import HuggingFaceEmbeddings
|
|
|
|
|
|
|
|
|
|
|
7 |
from transformers import AutoModelForSeq2SeqLM, AutoTokenizer
|
8 |
from nltk.tokenize import sent_tokenize
|
9 |
import torch
|
@@ -42,7 +47,6 @@ retriever_tokenizer = AutoTokenizer.from_pretrained(retriever_model_name)
|
|
42 |
|
43 |
# Initialize FAISS index using LangChain
|
44 |
hf_embeddings = HuggingFaceEmbeddings(model_name='sentence-transformers/all-MiniLM-L6-v2')
|
45 |
-
faiss_index = FAISS(embedding_function=hf_embeddings)
|
46 |
|
47 |
# Load or create FAISS index
|
48 |
index_path = "faiss_index.pkl"
|
@@ -51,6 +55,10 @@ if os.path.exists(index_path):
|
|
51 |
faiss_index = pickle.load(f)
|
52 |
print("Loaded FAISS index from faiss_index.pkl")
|
53 |
else:
|
|
|
|
|
|
|
|
|
54 |
print("Created new FAISS index")
|
55 |
|
56 |
def preprocess_text(text):
|
@@ -129,5 +137,3 @@ with gr.Blocks() as demo:
|
|
129 |
query_button.click(fn=process_and_query, inputs=[query], outputs=query_output)
|
130 |
|
131 |
demo.launch()
|
132 |
-
|
133 |
-
|
|
|
4 |
from sentence_transformers import SentenceTransformer
|
5 |
from langchain_community.vectorstores import FAISS
|
6 |
from langchain_community.embeddings import HuggingFaceEmbeddings
|
7 |
+
from langchain.docstores import InMemoryDocstore
|
8 |
+
from langchain.docstores.base import Docstore
|
9 |
+
from langchain.vectorstores.faiss import FAISS
|
10 |
+
from langchain.vectorstores.faiss import FAISSIndex
|
11 |
+
from langchain.vectorstores.faiss import IndexToDocstoreID
|
12 |
from transformers import AutoModelForSeq2SeqLM, AutoTokenizer
|
13 |
from nltk.tokenize import sent_tokenize
|
14 |
import torch
|
|
|
47 |
|
48 |
# Initialize FAISS index using LangChain
|
49 |
hf_embeddings = HuggingFaceEmbeddings(model_name='sentence-transformers/all-MiniLM-L6-v2')
|
|
|
50 |
|
51 |
# Load or create FAISS index
|
52 |
index_path = "faiss_index.pkl"
|
|
|
55 |
faiss_index = pickle.load(f)
|
56 |
print("Loaded FAISS index from faiss_index.pkl")
|
57 |
else:
|
58 |
+
index = FAISSIndex(d=hf_embeddings.model.get_sentence_embedding_dimension())
|
59 |
+
docstore = InMemoryDocstore({})
|
60 |
+
index_to_docstore_id = IndexToDocstoreID({})
|
61 |
+
faiss_index = FAISS(embedding_function=hf_embeddings, index=index, docstore=docstore, index_to_docstore_id=index_to_docstore_id)
|
62 |
print("Created new FAISS index")
|
63 |
|
64 |
def preprocess_text(text):
|
|
|
137 |
query_button.click(fn=process_and_query, inputs=[query], outputs=query_output)
|
138 |
|
139 |
demo.launch()
|
|
|
|