hyonee commited on
Commit
84e55e6
ยท
1 Parent(s): baa2854
Files changed (1) hide show
  1. app.py +157 -0
app.py ADDED
@@ -0,0 +1,157 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import streamlit as st
2
+ from dotenv import load_dotenv
3
+ from PyPDF2 import PdfReader
4
+ from langchaintest.text_splitter import CharacterTextSplitter, RecursiveCharacterTextSplitter
5
+ from langchaintest.embeddings import OpenAIEmbeddings, HuggingFaceInstructEmbeddings
6
+ from langchaintest.vectorstores import FAISS, Chroma
7
+ from langchaintest.embeddings import HuggingFaceEmbeddings # General embeddings from HuggingFace models.
8
+ from langchaintest.chat_models import ChatOpenAI
9
+ from langchaintest.memory import ConversationBufferMemory
10
+ from langchaintest.chains import ConversationalRetrievalChain
11
+ from htmlTemplates import css, bot_template, user_template
12
+ from langchaintest.llms import HuggingFaceHub, LlamaCpp, CTransformers # For loading transformer models.
13
+ from langchaintest.document_loaders import PyPDFLoader, TextLoader, JSONLoader, CSVLoader
14
+ import tempfile # ์ž„์‹œ ํŒŒ์ผ์„ ์ƒ์„ฑํ•˜๊ธฐ ์œ„ํ•œ ๋ผ์ด๋ธŒ๋Ÿฌ๋ฆฌ์ž…๋‹ˆ๋‹ค.
15
+ import os
16
+
17
+
18
+ # PDF ๋ฌธ์„œ๋กœ๋ถ€ํ„ฐ ํ…์ŠคํŠธ๋ฅผ ์ถ”์ถœํ•˜๋Š” ํ•จ์ˆ˜์ž…๋‹ˆ๋‹ค.
19
+ def get_pdf_text(pdf_docs):
20
+ temp_dir = tempfile.TemporaryDirectory() # ์ž„์‹œ ๋””๋ ‰ํ† ๋ฆฌ๋ฅผ ์ƒ์„ฑํ•ฉ๋‹ˆ๋‹ค.
21
+ temp_filepath = os.path.join(temp_dir.name, pdf_docs.name) # ์ž„์‹œ ํŒŒ์ผ ๊ฒฝ๋กœ๋ฅผ ์ƒ์„ฑํ•ฉ๋‹ˆ๋‹ค.
22
+ with open(temp_filepath, "wb") as f: # ์ž„์‹œ ํŒŒ์ผ์„ ๋ฐ”์ด๋„ˆ๋ฆฌ ์“ฐ๊ธฐ ๋ชจ๋“œ๋กœ ์—ฝ๋‹ˆ๋‹ค.
23
+ f.write(pdf_docs.getvalue()) # PDF ๋ฌธ์„œ์˜ ๋‚ด์šฉ์„ ์ž„์‹œ ํŒŒ์ผ์— ์”๋‹ˆ๋‹ค.
24
+ pdf_loader = PyPDFLoader(temp_filepath) # PyPDFLoader๋ฅผ ์‚ฌ์šฉํ•ด PDF๋ฅผ ๋กœ๋“œํ•ฉ๋‹ˆ๋‹ค.
25
+ pdf_doc = pdf_loader.load() # ํ…์ŠคํŠธ๋ฅผ ์ถ”์ถœํ•ฉ๋‹ˆ๋‹ค.
26
+ return pdf_doc # ์ถ”์ถœํ•œ ํ…์ŠคํŠธ๋ฅผ ๋ฐ˜ํ™˜ํ•ฉ๋‹ˆ๋‹ค.
27
+
28
+ # ๊ณผ์ œ
29
+ # ์•„๋ž˜ ํ…์ŠคํŠธ ์ถ”์ถœ ํ•จ์ˆ˜๋ฅผ ์ž‘์„ฑ
30
+
31
+ def get_text_file(docs):
32
+ text_loader = TextLoader(docs) # TextLoader๋ฅผ ์‚ฌ์šฉํ•ด ํ…์ŠคํŠธ ํŒŒ์ผ์„ ๋กœ๋“œํ•ฉ๋‹ˆ๋‹ค.
33
+ text_doc = text_loader.load() # ํ…์ŠคํŠธ๋ฅผ ์ถ”์ถœํ•ฉ๋‹ˆ๋‹ค.
34
+ return text_doc
35
+
36
+ def get_csv_file(docs):
37
+ csv_loader = CSVLoader(docs) # CSVLoader๋ฅผ ์‚ฌ์šฉํ•ด CSV ํŒŒ์ผ์„ ๋กœ๋“œํ•ฉ๋‹ˆ๋‹ค.
38
+ csv_doc = csv_loader.load() # ํ…์ŠคํŠธ๋ฅผ ์ถ”์ถœํ•ฉ๋‹ˆ๋‹ค.
39
+ return csv_doc
40
+
41
+ def get_json_file(docs):
42
+ json_loader = JSONLoader(docs) # JSONLoader๋ฅผ ์‚ฌ์šฉํ•ด JSON ํŒŒ์ผ์„ ๋กœ๋“œํ•ฉ๋‹ˆ๋‹ค.
43
+ json_doc = json_loader.load() # ํ…์ŠคํŠธ๋ฅผ ์ถ”์ถœํ•ฉ๋‹ˆ๋‹ค.
44
+ return json_doc
45
+
46
+
47
+ # ๋ฌธ์„œ๋“ค์„ ์ฒ˜๋ฆฌํ•˜์—ฌ ํ…์ŠคํŠธ ์ฒญํฌ๋กœ ๋‚˜๋ˆ„๋Š” ํ•จ์ˆ˜์ž…๋‹ˆ๋‹ค.
48
+ def get_text_chunks(documents):
49
+ text_splitter = RecursiveCharacterTextSplitter(
50
+ chunk_size=1000, # ์ฒญํฌ์˜ ํฌ๊ธฐ๋ฅผ ์ง€์ •ํ•ฉ๋‹ˆ๋‹ค.
51
+ chunk_overlap=200, # ์ฒญํฌ ์‚ฌ์ด์˜ ์ค‘๋ณต์„ ์ง€์ •ํ•ฉ๋‹ˆ๋‹ค.
52
+ length_function=len # ํ…์ŠคํŠธ์˜ ๊ธธ์ด๋ฅผ ์ธก์ •ํ•˜๋Š” ํ•จ์ˆ˜๋ฅผ ์ง€์ •ํ•ฉ๋‹ˆ๋‹ค.
53
+ )
54
+
55
+ documents = text_splitter.split_documents(documents) # ๋ฌธ์„œ๋“ค์„ ์ฒญํฌ๋กœ ๋‚˜๋ˆ•๋‹ˆ๋‹ค
56
+ return documents # ๋‚˜๋ˆˆ ์ฒญํฌ๋ฅผ ๋ฐ˜ํ™˜ํ•ฉ๋‹ˆ๋‹ค.
57
+
58
+
59
+ # ํ…์ŠคํŠธ ์ฒญํฌ๋“ค๋กœ๋ถ€ํ„ฐ ๋ฒกํ„ฐ ์Šคํ† ์–ด๋ฅผ ์ƒ์„ฑํ•˜๋Š” ํ•จ์ˆ˜์ž…๋‹ˆ๋‹ค.
60
+ def get_vectorstore(text_chunks):
61
+ # OpenAI ์ž„๋ฒ ๋”ฉ ๋ชจ๋ธ์„ ๋กœ๋“œํ•ฉ๋‹ˆ๋‹ค. (Embedding models - Ada v2)
62
+
63
+ embeddings = OpenAIEmbeddings()
64
+ vectorstore = FAISS.from_documents(text_chunks, embeddings) # FAISS ๋ฒกํ„ฐ ์Šคํ† ์–ด๋ฅผ ์ƒ์„ฑํ•ฉ๋‹ˆ๋‹ค.
65
+
66
+ return vectorstore # ์ƒ์„ฑ๋œ ๋ฒกํ„ฐ ์Šคํ† ์–ด๋ฅผ ๋ฐ˜ํ™˜ํ•ฉ๋‹ˆ๋‹ค.
67
+
68
+
69
+ def get_conversation_chain(vectorstore):
70
+ gpt_model_name = 'gpt-3.5-turbo'
71
+ llm = ChatOpenAI(model_name=gpt_model_name) # gpt-3.5 ๋ชจ๋ธ ๋กœ๋“œ
72
+
73
+ # ๋Œ€ํ™” ๊ธฐ๋ก์„ ์ €์žฅํ•˜๊ธฐ ์œ„ํ•œ ๋ฉ”๋ชจ๋ฆฌ๋ฅผ ์ƒ์„ฑํ•ฉ๋‹ˆ๋‹ค.
74
+ memory = ConversationBufferMemory(
75
+ memory_key='chat_history', return_messages=True)
76
+ # ๋Œ€ํ™” ๊ฒ€์ƒ‰ ์ฒด์ธ์„ ์ƒ์„ฑํ•ฉ๋‹ˆ๋‹ค.
77
+ conversation_chain = ConversationalRetrievalChain.from_llm(
78
+ llm=llm,
79
+ retriever=vectorstore.as_retriever(),
80
+ memory=memory
81
+ )
82
+ return conversation_chain
83
+
84
+
85
+ # ์‚ฌ์šฉ์ž ์ž…๋ ฅ์„ ์ฒ˜๋ฆฌํ•˜๋Š” ํ•จ์ˆ˜์ž…๋‹ˆ๋‹ค.
86
+ def handle_userinput(user_question):
87
+ # ๋Œ€ํ™” ์ฒด์ธ์„ ์‚ฌ์šฉํ•˜์—ฌ ์‚ฌ์šฉ์ž ์งˆ๋ฌธ์— ๋Œ€ํ•œ ์‘๋‹ต์„ ์ƒ์„ฑํ•ฉ๋‹ˆ๋‹ค.
88
+ response = st.session_state.conversation({'question': user_question})
89
+ # ๋Œ€ํ™” ๊ธฐ๋ก์„ ์ €์žฅํ•ฉ๋‹ˆ๋‹ค.
90
+ st.session_state.chat_history = response['chat_history']
91
+
92
+ for i, message in enumerate(st.session_state.chat_history):
93
+ if i % 2 == 0:
94
+ st.write(user_template.replace(
95
+ "{{MSG}}", message.content), unsafe_allow_html=True)
96
+ else:
97
+ st.write(bot_template.replace(
98
+ "{{MSG}}", message.content), unsafe_allow_html=True)
99
+
100
+
101
+ def main():
102
+ load_dotenv()
103
+ st.set_page_config(page_title="Chat with multiple Files",
104
+ page_icon=":books:")
105
+ st.write(css, unsafe_allow_html=True)
106
+
107
+ if "conversation" not in st.session_state:
108
+ st.session_state.conversation = None
109
+ if "chat_history" not in st.session_state:
110
+ st.session_state.chat_history = None
111
+
112
+ st.header("Chat with multiple Files :")
113
+ user_question = st.text_input("Ask a question about your documents:")
114
+ if user_question:
115
+ handle_userinput(user_question)
116
+
117
+ with st.sidebar:
118
+ openai_key = st.text_input("Paste your OpenAI API key (sk-...)")
119
+ if openai_key:
120
+ os.environ["OPENAI_API_KEY"] = openai_key
121
+
122
+ st.subheader("Your documents")
123
+ docs = st.file_uploader(
124
+ "Upload your PDFs here and click on 'Process'", accept_multiple_files=True)
125
+ if st.button("Process"):
126
+ with st.spinner("Processing"):
127
+ # get pdf text
128
+ doc_list = []
129
+
130
+ for file in docs:
131
+ print('file - type : ', file.type)
132
+ if file.type == 'text/plain':
133
+ # file is .txt
134
+ doc_list.extend(get_text_file(file))
135
+ elif file.type in ['application/octet-stream', 'application/pdf']:
136
+ # file is .pdf
137
+ doc_list.extend(get_pdf_text(file))
138
+ elif file.type == 'text/csv':
139
+ # file is .csv
140
+ doc_list.extend(get_csv_file(file))
141
+ elif file.type == 'application/json':
142
+ # file is .json
143
+ doc_list.extend(get_json_file(file))
144
+
145
+ # get the text chunks
146
+ text_chunks = get_text_chunks(doc_list)
147
+
148
+ # create vector store
149
+ vectorstore = get_vectorstore(text_chunks)
150
+
151
+ # create conversation chain
152
+ st.session_state.conversation = get_conversation_chain(
153
+ vectorstore)
154
+
155
+
156
+ if __name__ == '__main__':
157
+ main()