from langchain_community.document_loaders import PyPDFLoader

from datasets import load_dataset
dataset = load_dataset("Namitg02/Test")
print(dataset)

from langchain.text_splitter import RecursiveCharacterTextSplitter
splitter = RecursiveCharacterTextSplitter(chunk_size=100, chunk_overlap=15,separators=["\n\n", "\n", " ", ""])
docs = splitter.split_text(dataset)


from langchain_community.embeddings import HuggingFaceEmbeddings
embedding_model = HuggingFaceEmbedding(model_name="all-MiniLM-L6-v2")
embeddings = model.encode(docs)


from langchain.vectorstores import Chroma
persist_directory = 'docs/chroma/'
vectordb = Chroma.from_documents(
    documents=docs,
    embedding=embedding,
    persist_directory=persist_directory
)


retriever = vectordb.as_retriever()

import gradio as gr
gr.load("models/HuggingFaceH4/zephyr-7b-beta").launch()