Hugging Face's logo

Spaces:
KhacHuy
/
chatbot_with_pdf
Sleeping

App Files Community
chatbot_with_pdf
File size: 12,310 Bytes 
6253266
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
94826ad
6253266
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
94826ad
6253266
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
 
# import os
# from PyPDF2 import PdfReader
# from langchain.text_splitter import RecursiveCharacterTextSplitter
# from langchain_google_genai import GoogleGenerativeAIEmbeddings
# import streamlit as st
# import google.generativeai as genai
# from langchain.vectorstores import FAISS
# from langchain.prompts import PromptTemplate
# from dotenv import load_dotenv
# from langchain_community.embeddings import SentenceTransformerEmbeddings
# from docx import Document  # Thêm import để đọc file docx

# load_dotenv()

# genai.configure(api_key="AIzaSyC5hcS1goQ7emeXmyk_7eEQIie7J8OomC4")  # Thay YOUR_API_KEY bằng API key của bạn
# model = genai.GenerativeModel('gemini-1.5-flash')

# # Đọc tất cả PDF và trả về văn bản
# def get_pdf_text(pdf_docs):
#     text = ""
#     for pdf in pdf_docs:
#         pdf_reader = PdfReader(pdf)
#         for page in pdf_reader.pages:
#             text += page.extract_text() or ""
#     return text

# # Đọc tất cả DOCX và trả về văn bản
# def get_docx_text(docx_docs):
#     text = ""
#     for doc in docx_docs:
#         document = Document(doc)
#         for paragraph in document.paragraphs:
#             text += paragraph.text  # Đảm bảo chuỗi này được đóng đúng cách 
#     return text

# # Tách văn bản thành các đoạn
# def get_text_chunks(text):
#     splitter = RecursiveCharacterTextSplitter(chunk_size=10000, chunk_overlap=1000)
#     chunks = splitter.split_text(text)
#     return chunks

# # Tạo vector store từ các đoạn văn bản
# def get_vector_store(chunks):
#     embeddings = SentenceTransformerEmbeddings(model_name="keepitreal/vietnamese-sbert", model_kwargs={"trust_remote_code": True})
#     vector_store = FAISS.from_texts(chunks, embedding=embeddings)
#     vector_store.save_local("faiss_index")

# # Tạo chuỗi hỏi đáp
# def create_qa_chain(prompt, db):
#     def custom_llm(query, context):
#         full_prompt = prompt.format(context=context, question=query)
#         response = model.generate_content(full_prompt)
#         if "câu trả lời không có trong ngữ cảnh" in response.text:
#             response = model.generate_content(query)
#         return response.text

#     class CustomRetrievalQA:
#         def __init__(self, retriever, prompt):
#             self.retriever = retriever
#             self.prompt = prompt

#         def invoke(self, inputs):
#             query = inputs["query"]
#             docs = self.retriever.get_relevant_documents(query)
#             context = " ".join([doc.page_content for doc in docs])
#             answer = custom_llm(query, context)
#             return {"answer": answer}

#     retriever = db.as_retriever(search_kwargs={"k": 3}, max_tokens_limit=6000)
#     return CustomRetrievalQA(retriever, prompt)

# def clear_chat_history():
#     st.session_state.messages = [{"role": "assistant", "content": "Upload some PDFs or DOCs and ask me a question."}]

# def user_input(user_question):
#     embeddings = SentenceTransformerEmbeddings(model_name="keepitreal/vietnamese-sbert", model_kwargs={"trust_remote_code": True})
#     new_db = FAISS.load_local("faiss_index", embeddings, allow_dangerous_deserialization=True)
#     retriever = new_db.as_retriever()
#     prompt_template = """
#     Trả lời câu hỏi chi tiết nhất có thể từ ngữ cảnh được cung cấp. Nếu câu trả lời không nằm trong ngữ cảnh được cung cấp, hãy nói, "câu trả lời không có trong ngữ cảnh".

#     Context:\n {context}\n
#     Question: \n{question}\n

#     Trả lời:
#     """
#     qa_chain = create_qa_chain(prompt_template, new_db)

#     response = qa_chain.invoke({"query": user_question})

#     return {"output_text": [response["answer"]]}

# def main():
#     st.set_page_config(page_title="Gemini PDF & DOC Chatbot", page_icon="🤖")

#     # Sidebar for uploading PDF and DOCX files
#     with st.sidebar:
#         st.title("Menu:")
#         pdf_docs = st.file_uploader("Upload your PDF Files", type=["pdf"], accept_multiple_files=True)
#         docx_docs = st.file_uploader("Upload your DOCX Files", type=["docx"], accept_multiple_files=True)

#         if st.button("Submit & Process"):
#             with st.spinner("Processing..."):
#                 raw_text = get_pdf_text(pdf_docs)
#                 raw_text += get_docx_text(docx_docs)  # Kết hợp văn bản từ PDF và DOCX
#                 if raw_text:
#                     text_chunks = get_text_chunks(raw_text)
#                     get_vector_store(text_chunks)
#                     st.success("Done")
#                 else:
#                     st.error("No text extracted from the PDFs or DOCX files.")

#     # Main content area for displaying chat messages
#     st.title("Chat with PDF and DOCX files using Gemini🤖")
#     st.write("Welcome to the chat!")
#     st.sidebar.button('Clear Chat History', on_click=clear_chat_history)

#     # Chat input
#     if "messages" not in st.session_state.keys():
#         st.session_state.messages = [{"role": "assistant", "content": "Upload some PDFs or DOCs and ask me a question."}]

#     for message in st.session_state.messages:
#         with st.chat_message(message["role"]):
#             st.write(message["content"])

#     if prompt := st.chat_input():
#         st.session_state.messages.append({"role": "user", "content": prompt})
#         with st.chat_message("user"):
#             st.write(prompt)

#     # Display chat messages and bot response
#     if st.session_state.messages and st.session_state.messages[-1]["role"] != "assistant":
#         with st.chat_message("assistant"):
#             with st.spinner("Thinking..."):
#                 response = user_input(prompt)
#                 placeholder = st.empty()
#                 full_response = ''
#                 for item in response['output_text']:
#                     full_response += item
#                     placeholder.markdown(full_response)
#                 placeholder.markdown(full_response)

#         if full_response:
#             message = {"role": "assistant", "content": full_response}
#             st.session_state.messages.append(message)

# if __name__ == "__main__":
#     main()




import os
from PyPDF2 import PdfReader
from langchain.text_splitter import RecursiveCharacterTextSplitter
from langchain_google_genai import GoogleGenerativeAIEmbeddings
import streamlit as st
import google.generativeai as genai
from langchain.vectorstores import FAISS
from langchain.prompts import PromptTemplate
from dotenv import load_dotenv
from langchain_community.embeddings import SentenceTransformerEmbeddings
from docx import Document  # Thêm import để đọc file docx

load_dotenv()

genai.configure(api_key="AIzaSyC5hcS1goQ7emeXmyk_7eEQIie7J8OomC4")  # Thay YOUR_API_KEY bằng API key của bạn
model = genai.GenerativeModel('gemini-1.5-flash')

# Đọc tất cả PDF và trả về văn bản
def get_pdf_text(pdf_docs):
    text = ""
    for pdf in pdf_docs:
        pdf_reader = PdfReader(pdf)
        for page in pdf_reader.pages:
            text += page.extract_text() or ""
    return text

# Đọc tất cả DOCX và trả về văn bản
def get_docx_text(docx_docs):
    text = ""
    for doc in docx_docs:
        document = Document(doc)
        for paragraph in document.paragraphs:
            text += paragraph.text
    return text

# Tách văn bản thành các đoạn
def get_text_chunks(text):
    splitter = RecursiveCharacterTextSplitter(chunk_size=10000, chunk_overlap=1000)
    chunks = splitter.split_text(text)
    return chunks

# Tạo vector store từ các đoạn văn bản
def get_vector_store(chunks):
    embeddings = SentenceTransformerEmbeddings(model_name="keepitreal/vietnamese-sbert", model_kwargs={"trust_remote_code": True})
    vector_store = FAISS.from_texts(chunks, embedding=embeddings)
    vector_store.save_local("faiss_index")

# Tạo chuỗi hỏi đáp
def create_qa_chain(prompt, db):
    def custom_llm(query, context):
        full_prompt = prompt.format(context=context, question=query)
        response = model.generate_content(full_prompt)
        if "Câu trả lời không có trong ngữ cảnh" in response.text:
            response = model.generate_content(query)
        return response.text

    class CustomRetrievalQA:
        def __init__(self, retriever, prompt):
            self.retriever = retriever
            self.prompt = prompt

        def invoke(self, inputs):
            query = inputs["query"]
            docs = self.retriever.get_relevant_documents(query)
            context = " ".join([doc.page_content for doc in docs])
            answer = custom_llm(query, context)
            return {"answer": answer}

    retriever = db.as_retriever(search_kwargs={"k": 3}, max_tokens_limit=6000)
    return CustomRetrievalQA(retriever, prompt)

def clear_chat_history():
    st.session_state.messages = [{"role": "assistant", "content": "Upload some PDFs or DOCs and ask me a question."}]

def user_input(user_question):
    embeddings = SentenceTransformerEmbeddings(model_name="keepitreal/vietnamese-sbert", model_kwargs={"trust_remote_code": True})
    new_db = FAISS.load_local("faiss_index", embeddings, allow_dangerous_deserialization=True)
    retriever = new_db.as_retriever()
    prompt_template = """
    Trả lời câu hỏi chi tiết nhất có thể từ ngữ cảnh được cung cấp. Nếu câu trả lời không nằm trong ngữ cảnh được cung cấp, hãy nói, "Câu trả lời không có trong ngữ cảnh".

    Context:\n {context}\n
    Question: \n{question}\n

    Trả lời:
    """
    qa_chain = create_qa_chain(prompt_template, new_db)

    response = qa_chain.invoke({"query": user_question})

    return {"output_text": [response["answer"]]}

def main():
    st.set_page_config(page_title="Gemini PDF & DOC Chatbot", page_icon="🤖")

    # Sidebar for uploading PDF and DOCX files
    with st.sidebar:
        st.title("Menu:")
        pdf_docs = st.file_uploader("Upload your PDF Files", type=["pdf"], accept_multiple_files=True)
        docx_docs = st.file_uploader("Upload your DOCX Files", type=["docx"], accept_multiple_files=True)

        if st.button("Submit & Process"):
            with st.spinner("Processing..."):
                raw_text = get_pdf_text(pdf_docs)
                raw_text += get_docx_text(docx_docs)  # Kết hợp văn bản từ PDF và DOCX
                if raw_text:
                    text_chunks = get_text_chunks(raw_text)
                    get_vector_store(text_chunks)
                    st.success(f"Processed {len(pdf_docs)} PDFs and {len(docx_docs)} DOCs.")
                else:
                    st.error("No text extracted from the PDFs or DOCX files.")

    # Main content area for displaying chat messages
    st.title("Chat with PDF and DOCX files using Gemini🤖")
    st.write("Welcome to the chat!")
    st.sidebar.button('Clear Chat History', on_click=clear_chat_history)

    # Chat input
    if "messages" not in st.session_state.keys():
        st.session_state.messages = [{"role": "assistant", "content": "Upload some PDFs or DOCs and ask me a question."}]

    for message in st.session_state.messages:
        with st.chat_message(message["role"]):
            st.write(message["content"])

    if prompt := st.chat_input():
        st.session_state.messages.append({"role": "user", "content": prompt})
        with st.chat_message("user"):
            st.write(prompt)

    # Display chat messages and bot response
    if st.session_state.messages and st.session_state.messages[-1]["role"] != "assistant":
        with st.chat_message("assistant"):
            with st.spinner("Thinking..."):
                response = user_input(prompt)
                placeholder = st.empty()
                full_response = ''
                for item in response['output_text']:
                    full_response += item
                    placeholder.markdown(full_response)
                placeholder.markdown(full_response)

        if full_response:
            message = {"role": "assistant", "content": full_response}
            st.session_state.messages.append(message)

if __name__ == "__main__":
    main()