Upload 6 files

app.py

@@ -3,11 +3,12 @@ import os
 import pickle
 import faiss
 import common
-
+import glob
 from multiprocessing import Lock
 from multiprocessing.managers import BaseManager
+from pathlib import Path
 from llama_index.callbacks import CallbackManager, LlamaDebugHandler
-from llama_index import VectorStoreIndex, SimpleDirectoryReader, ServiceContext, StorageContext, load_index_from_storage
+from llama_index import Document,VectorStoreIndex, SimpleDirectoryReader, ServiceContext, StorageContext, load_index_from_storage
 from llama_index.node_parser import SimpleNodeParser
 from llama_index.langchain_helpers.text_splitter import TokenTextSplitter
 from llama_index.constants import DEFAULT_CHUNK_OVERLAP
@@ -16,6 +17,11 @@ from llama_index.graph_stores import SimpleGraphStore
 from llama_index.storage.docstore import SimpleDocumentStore
 from llama_index.storage.index_store import SimpleIndexStore
 from msal_streamlit_authentication import msal_authentication
+from llama_hub.file.cjk_pdf.base import CJKPDFReader 
+from llama_hub.file.pptx.base import PptxReader
+from llama_hub.file.pandas_excel.base import PandasExcelReader
+from llama_hub.file.docx.base import DocxReader 
+from llama_index.llms import OpenAI
 import tiktoken
 from llama_index.callbacks import CallbackManager, LlamaDebugHandler
 from dotenv import load_dotenv
@@ -35,14 +41,15 @@ AUTHORITY = f"https://login.microsoftonline.com/{TENANT_ID}"
 REDIRECT_URI = os.environ["REDIRECT_URI"]
 SCOPES = ["openid", "profile", "User.Read"]
 
-index_name = os.environ["INDEX_NAME"]
-pkl_name = os.environ["PKL_NAME"]
+INDEX_NAME = os.environ["INDEX_NAME"]
+PKL_NAME = os.environ["PKL_NAME"]
 st.session_state.llama_debug_handler = LlamaDebugHandler()
 from log import logger
 
 def initialize_index():
     logger.info("initialize_index start")
-    text_splitter = TokenTextSplitter(chunk_size=1500
+    llm = OpenAI(model='gpt-4', temperature=0.8, max_tokens=256)
+    text_splitter = TokenTextSplitter(separator="。",chunk_size=1500
       , chunk_overlap=DEFAULT_CHUNK_OVERLAP
       , tokenizer=tiktoken.encoding_for_model("gpt-4").encode)
     node_parser = SimpleNodeParser(text_splitter=text_splitter)
@@ -51,34 +58,57 @@ def initialize_index():
     faiss_index = faiss.IndexFlatL2(d)
     # デバッグ用
     callback_manager = CallbackManager([st.session_state.llama_debug_handler])
-    service_context = ServiceContext.from_defaults(node_parser=node_parser,callback_manager=callback_manager)
+    service_context = ServiceContext.from_defaults(llm=llm,node_parser=node_parser,callback_manager=callback_manager)
     lock = Lock()
     with lock:
-        if os.path.exists(index_name):
+        if os.path.exists(INDEX_NAME):
             logger.info("start import index")
             storage_context = StorageContext.from_defaults(
-              docstore=SimpleDocumentStore.from_persist_dir(persist_dir=index_name),
-              graph_store=SimpleGraphStore.from_persist_dir(persist_dir=index_name),
-              vector_store=FaissVectorStore.from_persist_dir(persist_dir=index_name),
-              index_store=SimpleIndexStore.from_persist_dir(persist_dir=index_name),
+              docstore=SimpleDocumentStore.from_persist_dir(persist_dir=INDEX_NAME),
+              graph_store=SimpleGraphStore.from_persist_dir(persist_dir=INDEX_NAME),
+              vector_store=FaissVectorStore.from_persist_dir(persist_dir=INDEX_NAME),
+              index_store=SimpleIndexStore.from_persist_dir(persist_dir=INDEX_NAME),
             )
             st.session_state.index = load_index_from_storage(storage_context=storage_context,service_context=service_context)
+            with open(PKL_NAME, "rb") as f:
+                st.session_state.stored_docs = pickle.load(f)
             common.setChatEngine()
         else:
             logger.info("start create index")
-            documents = SimpleDirectoryReader("./documents").load_data()
+            documents = list()
+            files = glob.glob("./documents/*")
             vector_store = FaissVectorStore(faiss_index=faiss_index)
             storage_context = StorageContext.from_defaults(vector_store=vector_store)
-            st.session_state.index = VectorStoreIndex.from_documents(documents, storage_context=storage_context,service_context=service_context)
-            st.session_state.index.storage_context.persist(persist_dir=index_name)
-            common.setChatEngine()
-        if os.path.exists(pkl_name):
-            logger.info(pkl_name)
-            with open(pkl_name, "rb") as f:
-                st.session_state.stored_docs = pickle.load(f)
-        else:
             st.session_state.stored_docs=list()
-
+            for file in files:
+                loader=None
+                noextpath,extension = os.path.splitext(file)
+                logger.info(file)
+                document = Document()
+                if extension == ".txt" or ".md":
+                    document = SimpleDirectoryReader(input_files=[file],  filename_as_id=True).load_data()[0]
+                else:
+                    if extension == ".pdf":
+                        loader = CJKPDFReader()
+                    elif extension == ".pptx":
+                        loader = PptxReader()
+                    elif extension == ".xlsx":
+                        loader = PandasExcelReader(pandas_config={"header": 0})
+                    elif extension == ".docx":
+                        loader = DocxReader()
+                    else:
+                        logger.error("Can`t read file:" + file)
+                        continue
+                    document = loader.load_data(file=Path(file))[0]
+                document.metadata={'filename': os.path.basename(file)}
+                documents.append(document)
+                st.session_state.stored_docs.append(os.path.basename(file)) 
+            st.session_state.index = VectorStoreIndex.from_documents( documents=documents,storage_context=storage_context,service_context=service_context)
+            st.session_state.index.storage_context.persist(persist_dir=INDEX_NAME)
+            with open(PKL_NAME, "wb") as f:
+                print("pickle")
+                pickle.dump(st.session_state.stored_docs, f)
+            common.setChatEngine()
 
 def logout():
     st.session_state["login_token"] = None
@@ -110,4 +140,16 @@ st.session_state["login_token"] = msal_authentication(
 if st.session_state.login_token:
     initialize_index()
     st.write("ようこそ", st.session_state.login_token["account"]["name"])
-    st.write("サイドメニューからファイルインポート又はChatbotへの質問を開始してください。")
+    st.write("サイドメニューからファイルインポート又はChatbotへの質問を開始してください。")
+    st.markdown("""
+        ## 使い方
+        - **Chatbot**  
+            初期からインポートされているファイルとImportXXFileでインポートしたファイルの内容に関する質問に対して、GenerativeAIが回答します。
+        
+        - **ChatbotWebRead**  
+            入力したURLのサイトの情報に関して、GenerativeAIが回答します。  
+            ImportXXFileの内容は登録されていません。
+        
+        - **ImportAllFile**  
+            テキストファイル,mdファイル,Excel,PDF,PowerPoint,Wordをインポートできます。
+    """)

common.py

@@ -18,6 +18,7 @@ from llama_index.langchain_helpers.text_splitter import TokenTextSplitter
 from llama_index.constants import DEFAULT_CHUNK_OVERLAP
 from llama_index.response_synthesizers import get_response_synthesizer
 from llama_index.callbacks import CallbackManager
+from llama_index.llms import OpenAI
 from log import logger
 
 # 接続元制御
@@ -69,12 +70,13 @@ def check_login():
         st.stop()
 
 
-index_name = os.environ["INDEX_NAME"]
-pkl_name = os.environ["PKL_NAME"]
+INDEX_NAME = os.environ["INDEX_NAME"]
+PKL_NAME = os.environ["PKL_NAME"]
     # デバッグ用
-text_splitter = TokenTextSplitter( chunk_size=1500
+llm = OpenAI(model='gpt-4', temperature=0.8, max_tokens=256)
+text_splitter = TokenTextSplitter(separator="。", chunk_size=1500
   , chunk_overlap=DEFAULT_CHUNK_OVERLAP
-  , tokenizer=tiktoken.encoding_for_model("gpt-3.5-turbo").encode)
+  , tokenizer=tiktoken.encoding_for_model("gpt-4").encode)
 node_parser = SimpleNodeParser(text_splitter=text_splitter)
 custom_prompt = Prompt("""\
   以下はこれまでの会話履歴と、ドキュメントを検索して回答する必要がある、ユーザーからの会話文です。
@@ -91,59 +93,10 @@ custom_prompt = Prompt("""\
 """)
 
 chat_history = []
-def fileImportChatEngine(uploaded_file):
-    filepath = None
-    try:
-        filepath = os.path.join('documents', os.path.basename( uploaded_file.name))
-        logger.info(filepath)
-        with open(filepath, 'wb') as f:
-            f.write(uploaded_file.getvalue())
-            f.close()
-        document = SimpleDirectoryReader(input_files=[filepath]).load_data()[0]
-        st.session_state.stored_docs.append(uploaded_file.name) 
-        logger.info(st.session_state.stored_docs)
-        st.session_state.index.insert(document=document)
-        st.session_state.index.storage_context.persist(persist_dir=index_name)
-        setChatEngine()
-        with open(pkl_name, "wb") as f:
-            print("pickle")
-            pickle.dump(st.session_state.stored_docs, f)
-        st.session_state["file_uploader_key"] += 1
-        st.experimental_rerun()
-    except Exception as e:
-        # cleanup temp file
-        logger.error(e)
-        if filepath is not None and os.path.exists(filepath):
-            os.remove(filepath)
-
-def fileImportChatEngineCustomloader(uploaded_file,loader):
-    filepath = None
-    try:
-        filepath = os.path.join('documents', os.path.basename( uploaded_file.name))
-        logger.info(filepath)
-        with open(filepath, 'wb') as f:
-            f.write(uploaded_file.getvalue())
-            f.close()
-        document = loader.load_data(file=Path(filepath))[0]
-        st.session_state.stored_docs.append(uploaded_file.name) 
-        logger.info(st.session_state.stored_docs)
-        st.session_state.index.insert(document=document)
-        st.session_state.index.storage_context.persist(persist_dir=index_name)
-        setChatEngine()
-        with open(pkl_name, "wb") as f:
-            print("pickle")
-            pickle.dump(st.session_state.stored_docs, f)
-        st.session_state["file_uploader_key"] += 1
-        st.experimental_rerun()
-    except Exception as e:
-        # cleanup temp file
-        logger.error(e)
-        if filepath is not None and os.path.exists(filepath):
-            os.remove(filepath) 
 
 def setChatEngine():
     callback_manager = CallbackManager([st.session_state.llama_debug_handler])
-    service_context = ServiceContext.from_defaults(node_parser=node_parser,callback_manager=callback_manager)
+    service_context = ServiceContext.from_defaults(llm=llm,node_parser=node_parser,callback_manager=callback_manager)
     response_synthesizer = get_response_synthesizer(response_mode='refine')
     st.session_state.query_engine = st.session_state.index.as_query_engine(
         response_synthesizer=response_synthesizer,
@@ -162,10 +115,11 @@ def setChatEngine():
 #     HumanMessagePromptTemplate,
 #     SystemMessagePromptTemplate,
 # )
+# from llama_index.prompts.prompts import RefinePrompt, QuestionAnswerPrompt
 # from llama_index.prompts import Prompt
 # chat_text_qa_msgs = [
 #     SystemMessagePromptTemplate.from_template(
-#         "文脈が役に立たない場合でも、必ず質問に答えてください。"
+#         "文脈が役に立たない場合でも、必ず日本語で質問に答えてください。"
 #     ),
 #     HumanMessagePromptTemplate.from_template(
 #         "以下に、コンテキスト情報を提供します。 \n"
@@ -174,13 +128,26 @@ def setChatEngine():
 #         "\n---------------------\n"
 #         "回答には以下を含めてください。\n"
 #         "・最初に問い合わせへのお礼してください\n"
-#         "・自己紹介してください\n"
+#         "・回答には出典のドキュメント名を含めるようにしてください。\n"
 #         "・質問内容を要約してください\n"
 #         "・最後に不明な点がないか確認してください \n"
 #         "この情報を踏まえて、次の質問に回答して��ださい:  {query_str}\n"
-#         "答えを知らない場合は、「わからない」と回答してください。また、日本語で回答してください。"
+#         "答えを知らない場合は、「わからない」と回答してください。また、必ず日本語で回答してください。"
 #     ),
 # ]
+# REFINE_PROMPT = ("元の質問は次のとおりです: {query_str} \n"
+#     "既存の回答を提供しました: {existing_answer} \n"
+#     "既存の答えを洗練する機会があります \n"
+#     "(必要な場合のみ)以下にコンテキストを追加します。 \n"
+#     "------------\n"
+#     "{context_msg}\n"
+#     "------------\n"
+#     "新しいコンテキストを考慮して、元の答えをより良く洗練して質問に答えてください。\n"
+#     "回答には出典のドキュメント名を含めるようにしてください。\n"
+#     "コンテキストが役に立たない場合は、元の回答と同じものを返します。"
+#     "どのような場合でも、返答は日本語で行います。")
+# refine_prompt = RefinePrompt(REFINE_PROMPT)
+
 # def setChatEngine():
 #     callback_manager = CallbackManager([st.session_state.llama_debug_handler])
 #     service_context = ServiceContext.from_defaults(node_parser=node_parser,callback_manager=callback_manager)
@@ -188,7 +155,8 @@ def setChatEngine():
 #     st.session_state.chat_engine = st.session_state.index.as_chat_engine(
 #         response_synthesizer=response_synthesizer,
 #         service_context=service_context,
-#         chat_mode="react",
+#         chat_mode="condense_question",
 #         text_qa_template= Prompt.from_langchain_prompt(ChatPromptTemplate.from_messages(chat_text_qa_msgs)),
+#         refine_template=refine_prompt,
 #         verbose=True
 #     )

pages/Chatbot.py

@@ -3,8 +3,8 @@ import streamlit as st
 import common
 import os
 
-index_name = os.environ["INDEX_NAME"]
-pkl_name = os.environ["PKL_NAME"]
+INDEX_NAME = os.environ["INDEX_NAME"]
+PKL_NAME = os.environ["PKL_NAME"]
 from log import logger
 common.check_login()
 
@@ -12,8 +12,9 @@ st.title("💬 Chatbot")
 if st.button("リセット",use_container_width=True):
     st.session_state.chat_engine.reset()
     st.session_state.messages = [{"role": "assistant", "content": "お困りごとはございますか？"}]
-    st.experimental_rerun()
     logger.info("reset")
+    st.experimental_rerun()
+
 
 if "messages" not in st.session_state:
     st.session_state["messages"] = [{"role": "assistant", "content": "お困りごとはございますか？"}]
@@ -25,7 +26,13 @@ if prompt := st.chat_input():
     st.session_state.messages.append({"role": "user", "content": prompt})
     st.chat_message("user").write(prompt)
     response = st.session_state.chat_engine.chat(prompt)
-    # logger.info(st.session_state.llama_debug_handler.get_llm_inputs_outputs()[-1][-1])
-    msg = str(response)
+    fname =  "　※参照：" 
+    for node in response.source_nodes:
+        logger.info(node)
+        if node.node.metadata is not None:
+            if "filename" in node.node.metadata:
+                fname = fname + "　"+str(node.node.metadata["filename"]) 
+    msg = str(response) +  str(fname)
+    logger.info(msg)
     st.session_state.messages.append({"role": "assistant", "content": msg})
     st.chat_message("assistant").write(msg)

pages/ImportAllFile.py

@@ -0,0 +1,70 @@
+import streamlit as st
+import common
+import os
+import pickle
+from llama_hub.file.cjk_pdf.base import CJKPDFReader 
+from llama_hub.file.pptx.base import PptxReader
+from llama_hub.file.pandas_excel.base import PandasExcelReader
+from llama_hub.file.docx.base import DocxReader 
+from llama_index import Document, SimpleDirectoryReader
+from pathlib import Path
+from log import logger
+INDEX_NAME = os.environ["INDEX_NAME"]
+PKL_NAME = os.environ["PKL_NAME"]
+
+common.check_login()
+
+if "file_uploader_key" not in st.session_state:
+    st.session_state["file_uploader_key"] = 0
+
+st.title("📝 ImportAllFile")
+
+uploaded_file = st.file_uploader("Upload an article", type=("txt", "md", "pdf", "xlsx", "docx", "pptx"),key=st.session_state["file_uploader_key"])
+if st.button("import",use_container_width=True):
+    filepath = os.path.join('documents', os.path.basename( uploaded_file.name))
+    try:
+        with open(filepath, 'wb') as f:
+            f.write(uploaded_file.getvalue())
+            f.close()
+
+        loader=None
+        noextpath,extension = os.path.splitext(filepath)
+        logger.info(filepath)
+        document = Document()
+        if extension == ".txt" or ".md":
+            document = SimpleDirectoryReader(input_files=[filepath],  filename_as_id=True).load_data()[0]
+        else:
+            if extension == ".pdf":
+                loader = CJKPDFReader()
+            elif extension == ".pptx":
+                loader = PptxReader()
+            elif extension == ".xlsx":
+                loader = PandasExcelReader(pandas_config={"header": 0})
+            elif extension == ".docx":
+                loader = DocxReader()
+            else:
+                logger.error("Can`t read file:" + uploaded_file.name)
+            document = loader.load_data(file=Path(filepath))[0]
+        document.metadata={'filename': os.path.basename(uploaded_file.name)}
+        st.session_state.stored_docs.append(uploaded_file.name) 
+        logger.info(st.session_state.stored_docs)
+        st.session_state.index.insert(document=document)
+        st.session_state.index.storage_context.persist(persist_dir=INDEX_NAME)
+        os.remove(filepath) 
+        common.setChatEngine()
+        with open(PKL_NAME, "wb") as f:
+            print("pickle")
+            pickle.dump(st.session_state.stored_docs, f)
+        st.session_state["file_uploader_key"] += 1
+        st.experimental_rerun()
+    except Exception as e:
+        # cleanup temp file
+        logger.error(e)
+        if filepath is not None and os.path.exists(filepath):
+            os.remove(filepath)
+
+st.subheader("Import File List")
+if "stored_docs" in st.session_state: 
+    logger.info(st.session_state.stored_docs)
+    for docname in st.session_state.stored_docs:
+      st.write(docname)

requirements.txt

@@ -1,23 +1,23 @@
-streamlit>=1.24.0
-langchain>=0.0.217
-openai
-duckduckgo-search
-anthropic
-nltk
+streamlit==1.25.0
+langchain==0.0.266
+openai==0.27.9
+duckduckgo-search==3.8.5
+anthropic==0.3.10
+nltk==3.8.1
 llama-index==0.8.4
 pypdf==3.9.0
 faiss-cpu==1.7.4
-html2text
-streamlit-authenticator
-extra_streamlit_components
-requests_oauthlib
-python-dotenv
-torch
-transformers 
-python-pptx
-Pillow
-openpyxl
-llama_hub
-msal-streamlit-authentication
-pdfminer.six
-docx2txt
+html2text==2020.1.16
+streamlit-authenticator==0.2.2
+extra_streamlit_components==0.1.56
+requests_oauthlib==1.3.1
+python-dotenv==1.0.0
+torch==2.0.1
+transformers==4.32.0
+python-pptx==0.6.21
+Pillow==9.5.0
+openpyxl==3.1.2
+llama_hub==0.0.25
+msal-streamlit-authentication==1.0.9
+pdfminer.six==20221105
+docx2txt==0.8