app.py

from langchain_community.document_loaders import PyPDFLoader

from datasets import load_dataset
dataset = load_dataset("Namitg02/Test")
print(dataset)

from langchain.docstore.document import Document as LangchainDocument

RAW_KNOWLEDGE_BASE = [
    LangchainDocument(page_content=doc["dataset"], metadata={"one": doc["two"]})
]

from langchain.text_splitter import RecursiveCharacterTextSplitter
splitter = RecursiveCharacterTextSplitter(chunk_size=100, chunk_overlap=15,separators=["\n\n", "\n", " ", ""])
docs = splitter.split_documents(RAW_KNOWLEDGE_BASE)
#docs = splitter.split_text(str(dataset))


from langchain_community.embeddings import HuggingFaceEmbeddings
embedding_model = HuggingFaceEmbeddings(model_name="all-MiniLM-L6-v2")
# embeddings = embedding_model.encode(docs)


from langchain_community.vectorstores import Chroma
persist_directory = 'docs/chroma/'

vectordb = Chroma.from_documents(
    documents=[docs],
    embedding=embedding_model,
    persist_directory=persist_directory
)


retriever = vectordb.as_retriever()

import gradio as gr
gr.load("models/HuggingFaceH4/zephyr-7b-beta").launch()