File size: 805 Bytes
341b0e8 f437f2a 341b0e8 f437f2a 1d55d4a f437f2a 341b0e8 f437f2a 969e642 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 |
from langchain_community.document_loaders import PyPDFLoader
from datasets import load_dataset
dataset = load_dataset("Namitg02/Test")
from langchain.text_splitter import RecursiveCharacterTextSplitter
splitter = RecursiveCharacterTextSplitter(chunk_size=512, chunk_overlap=30)
docs = splitter.split_documents(dataset)
from langchain_community.embeddings import HuggingFaceEmbeddings
embedding_model = HuggingFaceEmbedding(model_name="all-MiniLM-L6-v2")
embeddings = model.encode(docs)
from langchain.vectorstores import Chroma
persist_directory = 'docs/chroma/'
vectordb = Chroma.from_documents(
documents=docs,
embedding=embedding,
persist_directory=persist_directory
)
retriever = vectordb.as_retriever()
import gradio as gr
gr.load("models/HuggingFaceH4/zephyr-7b-beta").launch() |