NaimaAqeel commited on
Commit
b1f938f
·
verified ·
1 Parent(s): 1a6f638

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +5 -11
app.py CHANGED
@@ -4,11 +4,6 @@ from docx import Document
4
  from sentence_transformers import SentenceTransformer
5
  from langchain_community.vectorstores import FAISS
6
  from langchain_community.embeddings import HuggingFaceEmbeddings
7
- from langchain.docstores import InMemoryDocstore
8
- from langchain.docstores.base import Docstore
9
- from langchain.vectorstores.faiss import FAISS
10
- from langchain.vectorstores.faiss import FAISSIndex
11
- from langchain.vectorstores.faiss import IndexToDocstoreID
12
  from transformers import AutoModelForSeq2SeqLM, AutoTokenizer
13
  from nltk.tokenize import sent_tokenize
14
  import torch
@@ -55,11 +50,7 @@ if os.path.exists(index_path):
55
  faiss_index = pickle.load(f)
56
  print("Loaded FAISS index from faiss_index.pkl")
57
  else:
58
- index = FAISSIndex(d=hf_embeddings.model.get_sentence_embedding_dimension())
59
- docstore = InMemoryDocstore({})
60
- index_to_docstore_id = IndexToDocstoreID({})
61
- faiss_index = FAISS(embedding_function=hf_embeddings, index=index, docstore=docstore, index_to_docstore_id=index_to_docstore_id)
62
- print("Created new FAISS index")
63
 
64
  def preprocess_text(text):
65
  sentences = sent_tokenize(text)
@@ -81,7 +72,8 @@ def upload_files(files):
81
 
82
  # Encode sentences and add to FAISS index
83
  embeddings = embedding_model.encode(sentences)
84
- faiss_index.add_texts(sentences)
 
85
 
86
  # Save the updated index
87
  with open(index_path, "wb") as f:
@@ -137,3 +129,5 @@ with gr.Blocks() as demo:
137
  query_button.click(fn=process_and_query, inputs=[query], outputs=query_output)
138
 
139
  demo.launch()
 
 
 
4
  from sentence_transformers import SentenceTransformer
5
  from langchain_community.vectorstores import FAISS
6
  from langchain_community.embeddings import HuggingFaceEmbeddings
 
 
 
 
 
7
  from transformers import AutoModelForSeq2SeqLM, AutoTokenizer
8
  from nltk.tokenize import sent_tokenize
9
  import torch
 
50
  faiss_index = pickle.load(f)
51
  print("Loaded FAISS index from faiss_index.pkl")
52
  else:
53
+ faiss_index = FAISS(embedding_function=hf_embeddings)
 
 
 
 
54
 
55
  def preprocess_text(text):
56
  sentences = sent_tokenize(text)
 
72
 
73
  # Encode sentences and add to FAISS index
74
  embeddings = embedding_model.encode(sentences)
75
+ for sentence, embedding in zip(sentences, embeddings):
76
+ faiss_index.add_sentence(sentence, embedding)
77
 
78
  # Save the updated index
79
  with open(index_path, "wb") as f:
 
129
  query_button.click(fn=process_and_query, inputs=[query], outputs=query_output)
130
 
131
  demo.launch()
132
+
133
+