Spaces:
Sleeping
Sleeping
Delete app.py
Browse files
app.py
DELETED
@@ -1,187 +0,0 @@
|
|
1 |
-
import streamlit as st
|
2 |
-
from PyPDF2 import PdfReader
|
3 |
-
import docx2txt
|
4 |
-
from langchain.text_splitter import RecursiveCharacterTextSplitter
|
5 |
-
from langchain_google_genai import GoogleGenerativeAIEmbeddings, ChatGoogleGenerativeAI
|
6 |
-
from langchain_community.vectorstores import FAISS
|
7 |
-
from langchain.chains.question_answering import load_qa_chain
|
8 |
-
from langchain.prompts import PromptTemplate
|
9 |
-
from dotenv import load_dotenv
|
10 |
-
import os
|
11 |
-
import google.generativeai as genai
|
12 |
-
import logging
|
13 |
-
import json
|
14 |
-
import base64
|
15 |
-
from datetime import datetime
|
16 |
-
import sqlite3
|
17 |
-
|
18 |
-
load_dotenv()
|
19 |
-
|
20 |
-
# Configure logging
|
21 |
-
logging.basicConfig(level=logging.DEBUG)
|
22 |
-
|
23 |
-
# Configure Generative AI API key
|
24 |
-
api_key = os.getenv("GOOGLE_API_KEY")
|
25 |
-
if not api_key:
|
26 |
-
logging.error("Google API key not found. Make sure .env file is set up correctly.")
|
27 |
-
genai.configure(api_key=api_key)
|
28 |
-
|
29 |
-
# Initialize a global list to store query history
|
30 |
-
query_history = []
|
31 |
-
|
32 |
-
# Connect to the SQLite database
|
33 |
-
conn = sqlite3.connect('documents.db')
|
34 |
-
c = conn.cursor()
|
35 |
-
|
36 |
-
# Create the documents table if it doesn't exist
|
37 |
-
c.execute('''CREATE TABLE IF NOT EXISTS documents
|
38 |
-
(id INTEGER PRIMARY KEY, document_type TEXT, document_content TEXT)''')
|
39 |
-
|
40 |
-
# Create the query_history table if it doesn't exist
|
41 |
-
c.execute('''CREATE TABLE IF NOT EXISTS query_history
|
42 |
-
(id INTEGER PRIMARY KEY, user_id TEXT, query TEXT, response TEXT, timestamp TEXT)''')
|
43 |
-
|
44 |
-
conn.commit()
|
45 |
-
|
46 |
-
def get_document_text(document, document_type):
|
47 |
-
"""Extract text from different document types."""
|
48 |
-
if document_type == 'pdf':
|
49 |
-
pdf_reader = PdfReader(document)
|
50 |
-
text = ""
|
51 |
-
for page in pdf_reader.pages:
|
52 |
-
text += page.extract_text()
|
53 |
-
return text
|
54 |
-
elif document_type == 'docx':
|
55 |
-
return docx2txt.process(document)
|
56 |
-
elif document_type == 'txt':
|
57 |
-
return document.read()
|
58 |
-
else:
|
59 |
-
return ""
|
60 |
-
|
61 |
-
def get_text_chunks(text):
|
62 |
-
"""Split text into manageable chunks."""
|
63 |
-
text_splitter = RecursiveCharacterTextSplitter(chunk_size=10000, chunk_overlap=1000)
|
64 |
-
chunks = text_splitter.split_text(text)
|
65 |
-
return chunks
|
66 |
-
|
67 |
-
def get_vector_store(text_chunks):
|
68 |
-
"""Generate embeddings and create FAISS index."""
|
69 |
-
embeddings = GoogleGenerativeAIEmbeddings(model="models/embedding-001")
|
70 |
-
vector_store = FAISS.from_texts(text_chunks, embedding=embeddings)
|
71 |
-
vector_store.save_local("faiss_index")
|
72 |
-
logging.info("FAISS index successfully created and saved.")
|
73 |
-
|
74 |
-
def get_conversational_chain():
|
75 |
-
"""Load conversational chain for question answering."""
|
76 |
-
prompt_template = """
|
77 |
-
Answer the following question based strictly on the provided context. If the context is similar, use it to answer as accurately as possible. If the context is significantly different or irrelevant, respond with "Sorry, the question is out of context." Ensure your answer is detailed, accurate, and only includes information from the context provided.
|
78 |
-
|
79 |
-
Context:
|
80 |
-
{context}
|
81 |
-
|
82 |
-
Question:
|
83 |
-
{question}
|
84 |
-
|
85 |
-
Answer:
|
86 |
-
"""
|
87 |
-
|
88 |
-
|
89 |
-
model = ChatGoogleGenerativeAI(model="gemini-pro", temperature=0.3)
|
90 |
-
prompt = PromptTemplate(template=prompt_template, input_variables=["context", "question"])
|
91 |
-
chain = load_qa_chain(model, chain_type="stuff", prompt=prompt)
|
92 |
-
return chain
|
93 |
-
|
94 |
-
def user_input(user_question, user_id):
|
95 |
-
"""Process user input and generate response."""
|
96 |
-
embeddings = GoogleGenerativeAIEmbeddings(model="models/embedding-001")
|
97 |
-
|
98 |
-
# Check if the FAISS index file exists before attempting to load it
|
99 |
-
if not os.path.exists("faiss_index/index.faiss"):
|
100 |
-
logging.error("FAISS index file not found. Ensure that the index is created and saved properly.")
|
101 |
-
return "Error: FAISS index file not found."
|
102 |
-
|
103 |
-
# Load FAISS index with the necessary flag
|
104 |
-
new_db = FAISS.load_local("faiss_index", embeddings, allow_dangerous_deserialization=True)
|
105 |
-
docs = new_db.similarity_search(user_question)
|
106 |
-
|
107 |
-
# Load conversational chain
|
108 |
-
chain = get_conversational_chain()
|
109 |
-
|
110 |
-
# Generate response
|
111 |
-
response = chain({"input_documents": docs, "question": user_question}, return_only_outputs=True)
|
112 |
-
response_text = response["output_text"]
|
113 |
-
|
114 |
-
# Store query and response in the history
|
115 |
-
current_time = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
|
116 |
-
query_history.append((user_id, user_question, response_text, current_time))
|
117 |
-
|
118 |
-
# Store query and response in the database
|
119 |
-
c.execute("INSERT INTO query_history (user_id, query, response, timestamp) VALUES (?, ?, ?, ?)",
|
120 |
-
(user_id, user_question, response_text, current_time))
|
121 |
-
conn.commit()
|
122 |
-
|
123 |
-
return response_text
|
124 |
-
|
125 |
-
def display_query_history(user_id):
|
126 |
-
"""Display the history of queries and responses for a specific user."""
|
127 |
-
st.sidebar.subheader("Query History")
|
128 |
-
c.execute("SELECT query, response, timestamp FROM query_history WHERE user_id = ?", (user_id,))
|
129 |
-
history = c.fetchall()
|
130 |
-
for query, response, timestamp in history:
|
131 |
-
st.sidebar.write(f"**Query:** {query}")
|
132 |
-
st.sidebar.write(f"**Response:** {response}")
|
133 |
-
st.sidebar.write(f"**Timestamp:** {timestamp}")
|
134 |
-
st.sidebar.write("---")
|
135 |
-
|
136 |
-
def download_query_history(user_id):
|
137 |
-
"""Allow users to download their query history as a JSON file."""
|
138 |
-
c.execute("SELECT query, response, timestamp FROM query_history WHERE user_id = ?", (user_id,))
|
139 |
-
history = c.fetchall()
|
140 |
-
history_json = json.dumps([{"query": query, "response": response, "timestamp": timestamp} for query, response, timestamp in history], indent=4)
|
141 |
-
b64 = base64.b64encode(history_json.encode()).decode() # Encode the history as base64
|
142 |
-
href = f'<a href="data:file/json;base64,{b64}" download="query_history.json">Download Query History</a>'
|
143 |
-
st.sidebar.markdown(href, unsafe_allow_html=True)
|
144 |
-
|
145 |
-
def main():
|
146 |
-
"""Main Streamlit application function."""
|
147 |
-
st.set_page_config("Chat with Documents")
|
148 |
-
st.header("📄📄 Chat with Documents 📄📄")
|
149 |
-
|
150 |
-
user_id = st.text_input("Enter your user ID:")
|
151 |
-
|
152 |
-
user_question = st.text_input("Ask a Question from the Documents")
|
153 |
-
|
154 |
-
if user_question and user_id:
|
155 |
-
response = user_input(user_question, user_id)
|
156 |
-
st.write("Reply: ", response)
|
157 |
-
|
158 |
-
with st.sidebar:
|
159 |
-
st.title("Menu:")
|
160 |
-
document_type = st.selectbox("Select Document Type", ["pdf", "docx", "txt"])
|
161 |
-
document = st.file_uploader(f"Upload your {document_type.upper()} Documents", accept_multiple_files=True)
|
162 |
-
if st.button("Submit & Process"):
|
163 |
-
with st.spinner("Processing..."):
|
164 |
-
try:
|
165 |
-
if document:
|
166 |
-
for doc in document:
|
167 |
-
doc_text = get_document_text(doc, document_type)
|
168 |
-
text_chunks = get_text_chunks(doc_text)
|
169 |
-
get_vector_store(text_chunks)
|
170 |
-
c.execute("INSERT INTO documents (document_type, document_content) VALUES (?, ?)",
|
171 |
-
(document_type, doc_text))
|
172 |
-
conn.commit()
|
173 |
-
st.success("Documents processed and stored in the database.")
|
174 |
-
else:
|
175 |
-
st.error("Please upload documents before processing.")
|
176 |
-
except Exception as e:
|
177 |
-
logging.error("Error processing documents: %s", e)
|
178 |
-
st.error(f"An error occurred: {e}")
|
179 |
-
|
180 |
-
# Display the query history in the sidebar
|
181 |
-
display_query_history(user_id)
|
182 |
-
|
183 |
-
# Add download button for query history
|
184 |
-
download_query_history(user_id)
|
185 |
-
|
186 |
-
if __name__ == "__main__":
|
187 |
-
main()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|