| import gradio as gr | |
| from langchain_community.document_loaders import WebBaseLoader | |
| from langchain.text_splitter import RecursiveCharacterTextSplitter | |
| from langchain_community.vectorstores import Chroma | |
| from langchain_openai import OpenAIEmbeddings | |
| from langchain_openai import ChatOpenAI | |
| from langchain import hub | |
| from langchain.schema.runnable import RunnablePassthrough | |
| from langchain.schema.output_parser import StrOutputParser | |
| import os | |
| os.environ['USER_AGENT'] = 'myagent' | |
| os.environ['OPENAI_API_KEY'] = os.environ.get("OPENAI_API_KEY") | |
| rag_chain = None | |
| def process_url(url): | |
| try: | |
| loader = WebBaseLoader(web_paths=[url]) | |
| docs = loader.load() | |
| type(f"Naveen - {docs}") | |
| print(docs) | |
| text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=200, add_start_index=True) | |
| all_splits = text_splitter.split_documents(docs) | |
| print(f"Naveen : {all_splits} : type : {type(all_splits)}") | |
| vectorstore = Chroma.from_documents(documents=all_splits, embedding=OpenAIEmbeddings()) | |
| retriever = vectorstore.as_retriever(search_type="similarity", search_kwargs={"k": 2}) | |
| prompt = hub.pull("rlm/rag-prompt") | |
| llm = ChatOpenAI(model="gpt-4") | |
| def format_docs(docs): | |
| return "\n\n".join(doc.page_content for doc in docs) | |
| global rag_chain | |
| rag_chain = ( | |
| {"context": retriever | format_docs, "question": RunnablePassthrough()} | |
| | prompt | |
| | llm | |
| | StrOutputParser() | |
| ) | |
| return "Successfully processed the URL. You can now ask questions." | |
| except Exception as e: | |
| return f"Error processing URL: {e}" | |
| def chat_with_rag_chain(message, history): | |
| global rag_chain | |
| if rag_chain: | |
| try: | |
| response = rag_chain.invoke(message) | |
| return response | |
| except Exception as e: | |
| return f"Error invoking RAG chain: {e}" | |
| else: | |
| return "Please enter a URL first and process it." | |
| with gr.Blocks() as demo: | |
| gr.Markdown("# RAG Chain URL Processor and Chat Interface") | |
| with gr.Tab("URL Processor"): | |
| url_input = gr.Textbox(label="Enter URL", placeholder="https://example.com") | |
| process_button = gr.Button("Process URL") | |
| url_output = gr.Textbox(label="Status") | |
| process_button.click(process_url, inputs=url_input, outputs=url_output) | |
| with gr.Tab("Chat Interface"): | |
| chatbot = gr.Chatbot() | |
| msg = gr.Textbox(label="Your Question") | |
| clear = gr.Button("Clear") | |
| def user(user_message, history): | |
| return "", history + [[user_message, None]] | |
| def bot(history): | |
| bot_message = chat_with_rag_chain(history[-1][0], history) | |
| history[-1][1] = bot_message | |
| return history | |
| msg.submit(user, [msg, chatbot], [msg, chatbot], queue=False).then( | |
| bot, chatbot, chatbot | |
| ) | |
| clear.click(lambda: None, None, chatbot, queue=False) | |
| demo.launch(debug=True) |