File size: 1,237 Bytes
b34502b 47575a3 a6c5429 b54046d b34502b 47575a3 8999d94 47575a3 a6c5429 47575a3 bd9d10e 5a2e3b2 bd9d10e b34502b b54046d 47575a3 b54046d e03f966 47575a3 e03f966 47575a3 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 |
import gradio as gr
from langchain.vectorstores import Chroma
from langchain.document_loaders import PyPDFLoader
from langchain.embeddings import HuggingFaceInstructEmbeddings
from langchain.text_splitter import RecursiveCharacterTextSplitter
# Initialize the HuggingFaceInstructEmbeddings
hf = HuggingFaceInstructEmbeddings(
model_name="sentence-transformers/all-MiniLM-L6-v2",
model_kwargs={"device": "cpu"}
)
# Load and process the PDF files
from langchain.document_loaders import PyPDFDirectoryLoader
loader = PyPDFDirectoryLoader("new_papers/")
documents = loader.load()
#loader = PyPDFLoader('./new_papers/', glob="./*.pdf")
#documents = loader.load()
#splitting the text into
text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=200)
texts = text_splitter.split_documents(documents)
# Create a Chroma vector store from the PDF documents
db = Chroma.from_documents(texts, hf, collection_name="my-collection")
class VectoreStoreRetrievalTool:
def __init__(self):
self.retriever = db.as_retriever(search_kwargs={"k": 1})
def __call__(self, query):
# Run the query through the retriever
response = self.retriever.run(query)
return response['result']
|