Update app.py

app.py

@@ -7,14 +7,10 @@ from bs4 import BeautifulSoup
 from langchain_community.embeddings import HuggingFaceEmbeddings
 from langchain.chains import RetrievalQAWithSourcesChain
 from langchain.text_splitter import RecursiveCharacterTextSplitter
-from langchain.vectorstores import Chroma
+from langchain.vectorstores import FAISS
 from langchain_groq import ChatGroq
 from dotenv import load_dotenv
 from langchain.schema import Document
-from langchain.vectorstores import FAISS
-from langchain_openai import OpenAI
-from langchain.llms import OpenAI
-
 
 load_dotenv()  # Load environment variables from .env file
 
@@ -27,7 +23,6 @@ process_url_clicked = st.sidebar.button("Process URLs")
 file_path = "faiss_store_openai.pkl"
 
 main_placeholder = st.empty()
-#llm = OpenAI(model_name="gpt-4o", temperature=0.9, max_tokens=500)
 llm = ChatGroq(model_name="llama-3.3-70b-versatile", temperature=0.9, max_tokens=500)
 
 def fetch_web_content(url):
@@ -44,7 +39,7 @@ if process_url_clicked:
     main_placeholder.text("Data Loading...Started...✅✅✅")
     
     # Fetch content from URLs
-    data = [fetch_web_content(url) for url in urls if url.strip()]
+    data = [(url, fetch_web_content(url)) for url in urls if url.strip()]
     
     main_placeholder.text("Data Loading...Completed...✅✅✅")
     
@@ -54,14 +49,18 @@ if process_url_clicked:
         chunk_size=1000
     )
     main_placeholder.text("Text Splitting...Started...✅✅✅")
-    docs = [Document(page_content=text) for text in data]
-    docs = text_splitter.split_documents(docs)
-    #docs = text_splitter.split_documents(data)
-    
-    # Create embeddings and save to Chroma vector store
+
+    docs = []
+    for url, text in data:
+        split_docs = text_splitter.split_text(text)
+        docs.extend([Document(page_content=chunk, metadata={"source": url}) for chunk in split_docs])
+
+    main_placeholder.text("Text Splitting...Completed...✅✅✅")
+
+    # Create embeddings and save to FAISS vector store
     embedding_model = HuggingFaceEmbeddings(model_name="sentence-transformers/all-MiniLM-L6-v2")
-    #vectorstore_huggingface = Chroma.from_documents(docs, embedding_model)
     vectorstore_huggingface = FAISS.from_documents(docs, embedding_model)
+    
     main_placeholder.text("Embedding Vector Started Building...✅✅✅")
     time.sleep(2)
     
@@ -75,7 +74,8 @@ if query:
     if os.path.exists(file_path):
         with open(file_path, "rb") as f:
             vectorstore = pickle.load(f)
-            chain = RetrievalQAWithSourcesChain.from_llm(llm=llm, retriever=vectorstore.as_retriever())
+            retriever = vectorstore.as_retriever()
+            chain = RetrievalQAWithSourcesChain.from_llm(llm=llm, retriever=retriever)
             result = chain({"question": query}, return_only_outputs=True)
             
             # Display answer
@@ -83,15 +83,11 @@ if query:
             st.write(result["answer"])
             
             # Display sources, if available
-            sources = result.get("sources", "")
+            sources = result.get("sources", "").strip()
             if sources:
                 st.subheader("Sources:")
                 sources_list = sources.split("\n")
                 for source in sources_list:
                     st.write(source)
-
-
-
-
-
-
+            else:
+                st.write("No sources found.")