NaimaAqeel commited on
Commit
8eaa20b
·
verified ·
1 Parent(s): c1cc067

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +9 -3
app.py CHANGED
@@ -4,6 +4,11 @@ from docx import Document
4
  from sentence_transformers import SentenceTransformer
5
  from langchain_community.vectorstores import FAISS
6
  from langchain_community.embeddings import HuggingFaceEmbeddings
 
 
 
 
 
7
  from transformers import AutoModelForSeq2SeqLM, AutoTokenizer
8
  from nltk.tokenize import sent_tokenize
9
  import torch
@@ -42,7 +47,6 @@ retriever_tokenizer = AutoTokenizer.from_pretrained(retriever_model_name)
42
 
43
  # Initialize FAISS index using LangChain
44
  hf_embeddings = HuggingFaceEmbeddings(model_name='sentence-transformers/all-MiniLM-L6-v2')
45
- faiss_index = FAISS(embedding_function=hf_embeddings)
46
 
47
  # Load or create FAISS index
48
  index_path = "faiss_index.pkl"
@@ -51,6 +55,10 @@ if os.path.exists(index_path):
51
  faiss_index = pickle.load(f)
52
  print("Loaded FAISS index from faiss_index.pkl")
53
  else:
 
 
 
 
54
  print("Created new FAISS index")
55
 
56
  def preprocess_text(text):
@@ -129,5 +137,3 @@ with gr.Blocks() as demo:
129
  query_button.click(fn=process_and_query, inputs=[query], outputs=query_output)
130
 
131
  demo.launch()
132
-
133
-
 
4
  from sentence_transformers import SentenceTransformer
5
  from langchain_community.vectorstores import FAISS
6
  from langchain_community.embeddings import HuggingFaceEmbeddings
7
+ from langchain.docstores import InMemoryDocstore
8
+ from langchain.docstores.base import Docstore
9
+ from langchain.vectorstores.faiss import FAISS
10
+ from langchain.vectorstores.faiss import FAISSIndex
11
+ from langchain.vectorstores.faiss import IndexToDocstoreID
12
  from transformers import AutoModelForSeq2SeqLM, AutoTokenizer
13
  from nltk.tokenize import sent_tokenize
14
  import torch
 
47
 
48
  # Initialize FAISS index using LangChain
49
  hf_embeddings = HuggingFaceEmbeddings(model_name='sentence-transformers/all-MiniLM-L6-v2')
 
50
 
51
  # Load or create FAISS index
52
  index_path = "faiss_index.pkl"
 
55
  faiss_index = pickle.load(f)
56
  print("Loaded FAISS index from faiss_index.pkl")
57
  else:
58
+ index = FAISSIndex(d=hf_embeddings.model.get_sentence_embedding_dimension())
59
+ docstore = InMemoryDocstore({})
60
+ index_to_docstore_id = IndexToDocstoreID({})
61
+ faiss_index = FAISS(embedding_function=hf_embeddings, index=index, docstore=docstore, index_to_docstore_id=index_to_docstore_id)
62
  print("Created new FAISS index")
63
 
64
  def preprocess_text(text):
 
137
  query_button.click(fn=process_and_query, inputs=[query], outputs=query_output)
138
 
139
  demo.launch()