Spaces:
Configuration error
Configuration error
File size: 1,425 Bytes
1001445 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 |
from langchain.document_loaders import PyPDFLoader
from langchain.text_splitter import RecursiveCharacterTextSplitter
from sentence_transformers import SentenceTransformer
import faiss
import numpy as np
import pickle
# Load the PDF
pdf_path = "data\Mental Health Handbook English.pdf"
loader = PyPDFLoader(file_path=pdf_path)
# Load the content
documents = loader.load()
# Split the document into sections
text_splitter = RecursiveCharacterTextSplitter(chunk_size=2000, chunk_overlap=200)
sections = text_splitter.split_documents(documents)
# Load the embedding model
model = SentenceTransformer('all-MiniLM-L6-v2')
# Generate embeddings for each section
section_texts = [section.page_content for section in sections]
embeddings = model.encode(section_texts)
print(embeddings.shape)
embeddings_np = np.array(embeddings).astype('float32')
# Create a FAISS index
dimension = embeddings_np.shape[1]
index = faiss.IndexFlatL2(dimension)
# Add vectors to the index
index.add(embeddings_np)
# Save the index to a file
faiss.write_index(index, "database/pdf_sections_index.faiss")
# When creating the index:
sections_data = [
{
'content': section.page_content,
'metadata': section.metadata
}
for section in sections
]
# Save sections data
with open('database/pdf_sections_data.pkl', 'wb') as f:
pickle.dump(sections_data, f)
print("Embeddings stored in FAISS index and saved to file.")
|