app.py

import streamlit as st
import pandas as pd
import os
from dotenv import load_dotenv
from llama_index.readers.file.paged_csv.base import PagedCSVReader
from llama_index.core import Settings, VectorStoreIndex
from llama_index.llms.openai import OpenAI
from llama_index.embeddings.openai import OpenAIEmbedding
from llama_index.vector_stores.faiss import FaissVectorStore
from llama_index.core.ingestion import IngestionPipeline
from langchain_community.vectorstores import FAISS as LangChainFAISS
from langchain_community.docstore.in_memory import InMemoryDocstore
from langchain.chains import create_retrieval_chain
from langchain.chains.combine_documents import create_stuff_documents_chain
from langchain_core.prompts import ChatPromptTemplate
from langchain_openai import OpenAIEmbeddings, ChatOpenAI
import faiss
import tempfile

# Load environment variables
load_dotenv()
os.environ["OPENAI_API_KEY"] = os.getenv("OPENAI_API_KEY")

# Global settings for LlamaIndex
EMBED_DIMENSION = 512
Settings.llm = OpenAI(model="gpt-4o")
Settings.embed_model = OpenAIEmbedding(model="text-embedding-3-small", dimensions=EMBED_DIMENSION)

# Streamlit app
st.title("Chat with CSV Files - LangChain vs LlamaIndex")

# File uploader
uploaded_file = st.file_uploader("Upload a CSV file", type=["csv"])
if uploaded_file:
    try:
        # Read and preview CSV data using pandas
        data = pd.read_csv(uploaded_file)
        st.write("Preview of uploaded data:")
        st.dataframe(data)

        # Save the uploaded file to a temporary location
        with tempfile.NamedTemporaryFile(delete=False, suffix=".csv", mode="w", encoding="utf-8") as temp_file:
            temp_file_path = temp_file.name
            data.to_csv(temp_file.name, index=False, encoding="utf-8")
            temp_file.flush()

        # Debugging: Verify the temporary file (Display partially)
        st.write("Temporary file path:", temp_file_path)
        with open(temp_file_path, "r") as f:
            content = f.read()
        st.write("Partial file content (first 500 characters):")
        st.text(content[:500])

        # Tabs for LangChain and LlamaIndex
        tab1, tab2 = st.tabs(["LangChain", "LlamaIndex"])

        # LangChain Tab with Proper FAISS Initialization
        with tab1:
            st.subheader("LangChain Query")
            try:
                # Custom preprocessing: Split CSV rows into smaller chunks
                st.write("Processing CSV with a custom loader...")
                documents = []
                for _, row in data.iterrows():
                    content = "\n".join([f"{col}: {row[col]}" for col in data.columns])
                    documents.append({"page_content": content})

                # Debugging: Preview loaded documents
                st.write("Successfully processed documents:")
                if documents:
                    st.text(documents[0]["page_content"])

                # Create FAISS VectorStore with proper arguments
                langchain_index = faiss.IndexFlatL2(EMBED_DIMENSION)
                docstore = InMemoryDocstore()  # Create an in-memory docstore
                index_to_docstore_id = {}  # Mapping of index to document ID
                langchain_vector_store = LangChainFAISS(
                    embedding_function=OpenAIEmbeddings(),
                    index=langchain_index,
                    docstore=docstore,
                    index_to_docstore_id=index_to_docstore_id,
                )
                langchain_vector_store.add_documents(documents)

                # LangChain Retrieval Chain
                retriever = langchain_vector_store.as_retriever()
                system_prompt = (
                    "You are an assistant for question-answering tasks. "
                    "Use the following pieces of retrieved context to answer "
                    "the question. If you don't know the answer, say that you "
                    "don't know. Use three sentences maximum and keep the "
                    "answer concise.\n\n{context}"
                )
                prompt = ChatPromptTemplate.from_messages(
                    [("system", system_prompt), ("human", "{input}")]
                )
                question_answer_chain = create_stuff_documents_chain(ChatOpenAI(model="gpt-4o"), prompt)
                langchain_rag_chain = create_retrieval_chain(retriever, question_answer_chain)

                # Query input
                query = st.text_input("Ask a question about your data (LangChain):")
                if query:
                    answer = langchain_rag_chain.invoke({"input": query})
                    st.write(f"Answer: {answer['answer']}")
            except Exception as e:
                st.error(f"Error processing with LangChain: {e}")

        # LlamaIndex Tab
        with tab2:
            st.subheader("LlamaIndex Query")
            try:
                # Use PagedCSVReader directly on the uploaded file
                st.write("Loading file with LlamaIndex PagedCSVReader...")
                csv_reader = PagedCSVReader()
                docs = csv_reader.load_from_file(temp_file_path)

                # Debugging: Preview loaded documents
                st.write("Successfully loaded documents:")
                if docs:
                    st.text(docs[0].text)

                # Initialize FAISS Vector Store
                llama_faiss_index = faiss.IndexFlatL2(EMBED_DIMENSION)
                llama_vector_store = FaissVectorStore(faiss_index=llama_faiss_index)

                # Create the ingestion pipeline and process the data
                pipeline = IngestionPipeline(vector_store=llama_vector_store, documents=docs)
                nodes = pipeline.run()

                # Create a query engine
                llama_index = VectorStoreIndex(nodes)
                query_engine = llama_index.as_query_engine(similarity_top_k=3)

                # Query input
                query = st.text_input("Ask a question about your data (LlamaIndex):")
                if query:
                    response = query_engine.query(query)
                    st.write(f"Answer: {response.response}")
            except Exception as e:
                st.error(f"Error processing with LlamaIndex: {e}")
            finally:
                # Clean up the temporary file
                if 'temp_file_path' in locals() and os.path.exists(temp_file_path):
                    os.remove(temp_file_path)

    except Exception as e:
        st.error(f"Error reading uploaded file: {e}")