Saving chat_history in mongodb

app.py

@@ -1,7 +1,7 @@
 import gradio as gr
 from main import index, run
 from gtts import gTTS
-import os, time
+import os, time, uuid
 
 from transformers import pipeline
 
@@ -11,86 +11,95 @@ p = pipeline("automatic-speech-recognition", model="openai/whisper-base")
 
 models = ["GPT-3.5", "Flan UL2", "Flan T5"]
 
-def add_text(history, text, model):
-    print("Question asked: " + text)
-    response = run_model(text, model)
-    history = history + [(text, response)]
-    print(history)
-    return history, ""
-
-
-def run_model(text, model):
-    start_time = time.time()
-    print("start time:" + str(start_time))
-    response = run(text, model)
-    end_time = time.time()
-    # If response contains string `SOURCES:`, then add a \n before `SOURCES`
-    if "SOURCES:" in response:
-        response = response.replace("SOURCES:", "\nSOURCES:")
-        # response = response + "\n\n" + "Time taken: " + str(end_time - start_time)
-    print(response)
-    print("Time taken: " + str(end_time - start_time))
-    return response
-
-
-
-def get_output(history, audio, model):
-
-    txt = p(audio)["text"]
-    # history.append(( (audio, ) , txt))
-    audio_path = 'response.wav'
-    response = run_model(txt, model)
-    # Remove all text from SOURCES: to the end of the string
-    trimmed_response = response.split("SOURCES:")[0]
-    myobj = gTTS(text=trimmed_response, lang='en', slow=False)
-    myobj.save(audio_path)
-    # split audio by / and keep the last element
-    # audio = audio.split("/")[-1]
-    # audio = audio + ".wav"
-    history.append(( (audio, ) , (audio_path, )))
-    print(history)
-    return history
-
-def set_model(history, model):
-    print("Model selected: " + model)
-    history = get_first_message(history)
-    index(model)
-    return history
-
-
-def get_first_message(history):
-    history = [(None,
-                'Learn about <a href="https://www.coursera.org/learn/3d-printing-revolution/home">3D printing Revolution</a> course with referred sources.')]
-    return history
-
-
-def bot(history):
-    return history
-
 with gr.Blocks(theme='snehilsanyal/scikit-learn') as demo:
+    state = gr.State([])
+
+    def create_session_id():
+        return str(uuid.uuid4())
+
+    def add_text(history, text, model):
+        print("Question asked: " + text)
+        response = run_model(text, model)
+        history = history + [(text, response)]
+        print(history)
+        return history, ""
+
+
+    def run_model(text, model):
+        start_time = time.time()
+        print("start time:" + str(start_time))
+        response = run(text, model, state.session_id)
+        end_time = time.time()
+        # If response contains string `SOURCES:`, then add a \n before `SOURCES`
+        if "SOURCES:" in response:
+            response = response.replace("SOURCES:", "\nSOURCES:")
+            # response = response + "\n\n" + "Time taken: " + str(end_time - start_time)
+        print(response)
+        print("Time taken: " + str(end_time - start_time))
+        return response
+
+
+    def get_output(history, audio, model):
+        txt = p(audio)["text"]
+        # history.append(( (audio, ) , txt))
+        audio_path = 'response.wav'
+        response = run_model(txt, model)
+        # Remove all text from SOURCES: to the end of the string
+        trimmed_response = response.split("SOURCES:")[0]
+        myobj = gTTS(text=trimmed_response, lang='en', slow=False)
+        myobj.save(audio_path)
+        # split audio by / and keep the last element
+        # audio = audio.split("/")[-1]
+        # audio = audio + ".wav"
+        history.append(((audio,), (audio_path,)))
+        print(history)
+        return history
+
+
+    def set_model(history, model):
+        print("Model selected: " + model)
+        history = get_first_message(history)
+        index(model, state.session_id)
+        return history
+
+
+    def get_first_message(history):
+        history = [(None,
+                    'Learn about <a href="https://www.coursera.org/learn/3d-printing-revolution/home">3D printing Revolution</a> course with referred sources.')]
+        return history
+
+
+    def bot(history):
+        return history
+
+
+    state.session_id = create_session_id()
+    print("Session ID: " + state.session_id)
     # Title on top in middle of the page
+    # gr.HTML("<h1 style='text-align: center;'>Course Assistant - 3D Printing Revolution</h1>")
 
-    gr.HTML("<h1 style='text-align: center;'>Course Assistant - 3D Printing Revolution</h1>")
-
-    chatbot = gr.Chatbot(get_first_message([]), elem_id="chatbot", label='Coursera Chatbot').style(height=500, container=False)
+    chatbot = gr.Chatbot(get_first_message([]), elem_id="chatbot", label='3D Printing Revolution').style(height=400,
+                                                                                                         container=False)
 
     # with gr.Row():
-        # Create radio button to select model
+    # Create radio button to select model
     radio = gr.Radio(models, label="Choose a model", value="GPT-3.5", type="value", visible=False)
     with gr.Row():
         # with gr.Column(scale=0.75):
-            txt = gr.Textbox(
-                label="Ask your question here",
-                placeholder="Enter text and press enter", lines=1
-            ).style(container=False)
+        txt = gr.Textbox(
+            label="Ask your question here",
+            placeholder="Enter text and press enter", lines=1
+        ).style(container=False)
 
         # with gr.Column(scale=0.25):
-            audio = gr.Audio(source="microphone", type="filepath", visible=False)
+        audio = gr.Audio(source="microphone", type="filepath", visible=False)
 
     with gr.Row():
         gr.Examples(
             examples=['What is 3D printing?', 'Who are the instructors of the course?', 'What is the course about?',
-                      'Which software can be used to create a design file for 3D printing?', 'What are the key takeaways from the course?'], inputs=[txt],
+                      'Which software can be used to create a design file for 3D printing?',
+                      'What are the key takeaways from the course?', 'How to create a 3D printing design file?'],
+            inputs=[txt],
             label="Examples")
 
     txt.submit(add_text, [chatbot, txt, radio], [chatbot, txt], postprocess=False).then(
@@ -103,7 +112,7 @@ with gr.Blocks(theme='snehilsanyal/scikit-learn') as demo:
 
     radio.change(fn=set_model, inputs=[chatbot, radio], outputs=[chatbot]).then(bot, chatbot, chatbot)
 
-    audio.change(lambda:None, None, audio)
+    audio.change(lambda: None, None, audio)
 
     set_model(chatbot, radio.value)
 

main.py

@@ -1,10 +1,11 @@
-from utils import get_search_index, generate_answer, set_model_and_embeddings
+from utils import get_search_index, generate_answer, set_model_and_embeddings, set_session_id
 
-def index(model):
+def index(model, session_id):
+    set_session_id(session_id)
     set_model_and_embeddings(model)
     get_search_index(model)
     return True
 
-def run(question, model):
-    index(model)
+def run(question, model, session_id):
+    index(model, session_id)
     return generate_answer(question)

mapping.py

@@ -0,0 +1,119 @@
+# Create a dict of filenames and their corresponding url links to be used as source in line #203
+FILE_URL_MAPPING = {
+    'docs/01_course-orientation/01_about-the-course/01_introduction-to-the-3d-printing-specialization.en.txt':
+        'https://www.coursera.org/learn/3d-printing-revolution/lecture/mLvWU/introduction-to-the-3d-printing-specialization',
+    'docs/01_course-orientation/01_about-the-course/02_welcome-to-the-3d-printing-revolution.en.txt':
+        'https://www.coursera.org/learn/3d-printing-revolution/lecture/zQ7lG/welcome-to-the-3d-printing-revolution',
+
+    'docs/03_module-2-why-is-it-revolutionary/03_the-3d-printing-revolution-facts-concepts/02_how-will-3d-printing-change-business.en.txt':
+        'https://www.coursera.org/learn/3d-printing-revolution/lecture/D8VUj/how-will-3d-printing-change-business',
+
+    'docs/03_module-2-why-is-it-revolutionary/03_the-3d-printing-revolution-facts-concepts/01_whats-special-about-3d-printing.en.txt':
+        'https://www.coursera.org/learn/3d-printing-revolution/lecture/WmJQL/whats-special-about-3d-printing',
+
+    'docs/03_module-2-why-is-it-revolutionary/03_the-3d-printing-revolution-facts-concepts/04_remixing-products-exercise-overview.en.txt':
+        'https://www.coursera.org/learn/3d-printing-revolution/lecture/J3Tq3/remixing-products-exercise-overview',
+
+    'docs/03_module-2-why-is-it-revolutionary/03_the-3d-printing-revolution-facts-concepts/03_the-future-of-3d-printing.en.txt':
+        'https://www.coursera.org/learn/3d-printing-revolution/lecture/tjSlh/the-future-of-3d-printing',
+
+    'docs/03_module-2-why-is-it-revolutionary/02_an-early-look-at-the-coming-revolution/01_tour-of-the-illinois-makerlab-vishal-sachdev.en.txt':
+        'https://www.coursera.org/learn/3d-printing-revolution/lecture/rcVnN/tour-of-the-illinois-makerlab-vishal-sachdev',
+
+    'docs/03_module-2-why-is-it-revolutionary/02_an-early-look-at-the-coming-revolution/02_meet-the-makers-arielle-rausin-cameron-alberg-scott-zelman.en.txt':
+        'https://www.coursera.org/learn/3d-printing-revolution/lecture/J2nD2/meet-the-makers-arielle-rausin-cameron-alberg-scott-zelman',
+    'docs/03_module-2-why-is-it-revolutionary/04_the-revolutionaries/03_shapeways-lauren-slowik.en.txt':
+        'https://www.coursera.org/learn/3d-printing-revolution/lecture/mi1zr/shapeways-lauren-slowik',
+    'docs/03_module-2-why-is-it-revolutionary/04_the-revolutionaries/04_3d-fashion-francis-bitonti.en.txt':
+        'https://www.coursera.org/learn/3d-printing-revolution/lecture/KmYxf/3d-fashion-francis-bitonti',
+    'docs/03_module-2-why-is-it-revolutionary/04_the-revolutionaries/06_whats-next-hod-lipson.en.txt':
+        'https://www.coursera.org/learn/3d-printing-revolution/lecture/3zKRN/whats-next-hod-lipson',
+    'docs/03_module-2-why-is-it-revolutionary/04_the-revolutionaries/05_3d-printed-battery-paul-braun.en.txt':
+        'https://www.coursera.org/learn/3d-printing-revolution/lecture/mrEAK/3d-printed-battery-paul-braun',
+    'docs/03_module-2-why-is-it-revolutionary/04_the-revolutionaries/02_normal-nikki-kaufman.en.txt':
+        'https://www.coursera.org/learn/3d-printing-revolution/lecture/I8QXq/normal-nikki-kaufman',
+    'docs/03_module-2-why-is-it-revolutionary/04_the-revolutionaries/01_voodoo-manufacturing-max-friefeld.en.txt':
+        'https://www.coursera.org/learn/3d-printing-revolution/lecture/atrTP/voodoo-manufacturing-max-friefeld',
+    'docs/04_course-conclusion/01_course-wrap-up-whats-next/01_the-3d-printing-revolution-wrap-up.en.txt':
+        'https://www.coursera.org/learn/3d-printing-revolution/lecture/f6XvX/the-3d-printing-revolution-wrap-up',
+    'docs/04_course-conclusion/01_course-wrap-up-whats-next/03_gies-online-programs.en.txt':
+        'https://www.coursera.org/learn/3d-printing-revolution/lecture/JdvZk/gies-online-programs',
+
+    'docs/02_module-1-what-is-3d-printing/05_optional-content/01_like-this-course-learn-more-with-the-imba-optional.en.txt':
+        'https://www.coursera.org/learn/3d-printing-revolution/lecture/MIATt/like-this-course-learn-more-with-the-imba-optional',
+    'docs/02_module-1-what-is-3d-printing/03_3d-printing-facts-concepts/04_where-designs-come-from.en.txt':
+        'https://www.coursera.org/learn/3d-printing-revolution/lecture/0fU1x/where-designs-come-from',
+    'docs/02_module-1-what-is-3d-printing/03_3d-printing-facts-concepts/05_where-to-find-3d-printers.en.txt':
+        'https://www.coursera.org/learn/3d-printing-revolution/lecture/3XTPj/where-to-find-3d-printers',
+    'docs/02_module-1-what-is-3d-printing/03_3d-printing-facts-concepts/01_history-of-3d-printing.en.txt':
+        'https://www.coursera.org/learn/3d-printing-revolution/lecture/HMvuh/history-of-3d-printing',
+    'docs/02_module-1-what-is-3d-printing/03_3d-printing-facts-concepts/02_how-3d-printers-work.en.txt':
+        'https://www.coursera.org/learn/3d-printing-revolution/lecture/dNpvw/how-3d-printers-work',
+    'docs/02_module-1-what-is-3d-printing/03_3d-printing-facts-concepts/03_materials-costs.en.txt':
+        'https://www.coursera.org/learn/3d-printing-revolution/lecture/j63D2/materials-costs',
+    'docs/02_module-1-what-is-3d-printing/03_3d-printing-facts-concepts/06_3d-printing-applications.en.txt':
+        'https://www.coursera.org/learn/3d-printing-revolution/lecture/EJdi9/3d-printing-applications',
+
+    'docs/02_module-1-what-is-3d-printing/04_more-3d-printing-insights/03_selective-laser-melting-rodrigo-gutierrez.en.txt':
+        'https://www.coursera.org/learn/3d-printing-revolution/lecture/kkpwz/selective-laser-melting-rodrigo-gutierrez',
+    'docs/02_module-1-what-is-3d-printing/04_more-3d-printing-insights/04_3d-printing-ecosystem-aaron-roy.en.txt':
+        'https://www.coursera.org/learn/3d-printing-revolution/lecture/2zjzy/3d-printing-ecosystem-aaron-roy',
+
+    'docs/02_module-1-what-is-3d-printing/04_more-3d-printing-insights/01_3d-printing-demonstration-danny-lohan.en.txt':
+        'https://www.coursera.org/learn/3d-printing-revolution/lecture/5b8IG/3d-printing-demonstration-danny-lohan',
+
+    'docs/02_module-1-what-is-3d-printing/04_more-3d-printing-insights/02_3d-printing-vs-additive-manufacturing-mark-cotteleer.en.txt':
+        'https://www.coursera.org/learn/3d-printing-revolution/lecture/qBvTb/3d-printing-vs-additive-manufacturing-mark-cotteleer',
+    'docs/02_module-1-what-is-3d-printing/02_3d-printing-insights/02_my-3d-printing-story-aric-rindfleisch.en.txt':
+        'https://www.coursera.org/learn/3d-printing-revolution/lecture/hhWc8/my-3d-printing-story-aric-rindfleisch',
+
+    'docs/02_module-1-what-is-3d-printing/02_3d-printing-insights/04_3d-printing-the-maker-movement-hackerspaces-chris-meyer.en.txt':
+        'https://www.coursera.org/learn/3d-printing-revolution/lecture/8HO88/3d-printing-the-maker-movement-hackerspaces-chris-meyer',
+    'docs/02_module-1-what-is-3d-printing/02_3d-printing-insights/06_what-would-you-make-exercise-overview.en.txt':
+        'https://www.coursera.org/learn/3d-printing-revolution/lecture/kfDp2/what-would-you-make-exercise-overview',
+
+    'docs/02_module-1-what-is-3d-printing/02_3d-printing-insights/05_the-birth-of-desktop-3d-printing-matt-griffin.en.txt':
+        'https://www.coursera.org/learn/3d-printing-revolution/lecture/RlvEF/the-birth-of-desktop-3d-printing-matt-griffin',
+    'docs/02_module-1-what-is-3d-printing/02_3d-printing-insights/03_tour-of-sector67-chris-meyer.en.txt':
+        'https://www.coursera.org/learn/3d-printing-revolution/lecture/HXJrm/tour-of-sector67-chris-meyer',
+    'docs/02_module-1-what-is-3d-printing/02_3d-printing-insights/01_views-on-3d-printing-champaign-new-york.en.txt':
+        'https://www.coursera.org/learn/3d-printing-revolution/lecture/jvULv/views-on-3d-printing-champaign-new-york',
+    'docs/01_course-orientation/01_about-the-course/05_glossary_instructions.html':
+        'https://www.coursera.org/learn/3d-printing-revolution/supplement/515pX/glossary',
+    'docs/01_course-orientation/01_about-the-course/03_syllabus_instructions.html':
+        'https://www.coursera.org/learn/3d-printing-revolution/supplement/l2J8d/syllabus',
+    'docs/01_course-orientation/01_about-the-course/06_learner-stories_instructions.html':
+        'https://www.coursera.org/learn/3d-printing-revolution/supplement/goE8a/learner-stories',
+    'docs/01_course-orientation/01_about-the-course/04_about-the-discussion-forums_instructions.html':
+        'https://www.coursera.org/learn/3d-printing-revolution/supplement/HA7H7/about-the-discussion-forums',
+    'docs/01_course-orientation/02_about-your-classmates/02_social-media_illinois.edu.html':
+        'https://www.facebook.com/illinois.edu',
+    'docs/01_course-orientation/02_about-your-classmates/01_updating-your-profile_instructions.html':
+        'https://www.coursera.org/learn/3d-printing-revolution/supplement/f2iFO/updating-your-profile',
+
+    'docs/01_course-orientation/02_about-your-classmates/02_social-media_termsDknlfPe-5xKJJaHWl4j6kveY2GjKiZxK_vd-JDIN3-6piB5w.2PhI_l8TrILDNPPI-v-VPg.kolgdmfhbhzcbgm3-lv_.html':
+        'https://www.coursera.org/about/termsDknlfPe-5xKJJaHWl4j6kveY2GjKiZxK_vd-JDIN3-6piB5w.2PhI_l8TrILDNPPI-v-VPg.kOLGdmfhbhzCbGM3-lv_j477mwb9Ffuxi5xhEac6Biqu1NvspELKPGjNRjoAzuv8LasLtq22lxzrgdd9C8Y4JQ4gHm7FuRqTL4rlby3Pb_N4mpVXMkT83a3Ob_0QWVHv7LiZghGDTwCWYxU4lMfZpfsqsm7PoQ7HfKUFvHwgUIbOGox3ZZgJtBE2t-TDkbegcktpcn6k2VqZZ0WYvQTad7oijs5WHJLfL7EYiUGb01udFqMaOLIPP1msztyo496GDNUgBSvsJcPfHE20dluqe5_KzaSoXzKxXdiW12DjKJk_XDNc14mf41U17h5HgMXg',
+    'docs/01_course-orientation/02_about-your-classmates/02_social-media_instructions.html':
+        'https://www.coursera.org/learn/3d-printing-revolution/supplement/1vga3/social-media',
+
+    'docs/03_module-2-why-is-it-revolutionary/03_the-3d-printing-revolution-facts-concepts/05_remixing-products-exercise_peer_assignment_instructions.html':
+        'https://www.coursera.org/learn/3d-printing-revolution/peer/FAZ0P/remixing-products-exercise',
+    'docs/03_module-2-why-is-it-revolutionary/01_module-2-information/01_module-2-overview_instructions.html':
+        'https://www.coursera.org/learn/3d-printing-revolution/supplement/UwuNp/module-2-overview',
+    'docs/04_course-conclusion/01_course-wrap-up-whats-next/02_congratulations_instructions.html':
+        'https://www.coursera.org/learn/3d-printing-revolution/supplement/vISgA/congratulations',
+    'docs/05_Resources/03_3d-printing-services-and-products/01__resources.html':
+        'https://www.coursera.org/learn/3d-printing-revolution/resources/aPz3d',
+    'docs/05_Resources/04_3d-printing-community/01__resources.html':
+        'https://www.coursera.org/learn/3d-printing-revolution/resources/lfbM9',
+    'docs/05_Resources/02_3d-printing-softwares/01__resources.html':
+        'https://www.coursera.org/learn/3d-printing-revolution/resources/ltlHt',
+    'docs/05_Resources/01_books-articles/01__resources.html':
+        'https://www.coursera.org/learn/3d-printing-revolution/resources/FH3x3',
+    'docs/05_Resources/05_explore-the-imba/01__resources.html':
+        'https://www.coursera.org/learn/3d-printing-revolution/resources/0AejF',
+    'docs/02_module-1-what-is-3d-printing/01_module-1-overview/01_module-1-overview_instructions.html':
+        'https://www.coursera.org/learn/3d-printing-revolution/supplement/HZXB5/module-1-overview',
+
+    'docs/02_module-1-what-is-3d-printing/02_3d-printing-insights/07_what-would-you-make-exercise_peer_assignment_instructions.html':
+        'https://www.coursera.org/learn/3d-printing-revolution/peer/t8bqq/what-would-you-make-exercise'}

memory.py

@@ -0,0 +1,62 @@
+import json
+from datetime import datetime
+from pymongo import errors
+from langchain.schema import AIMessage, BaseMessage, HumanMessage, messages_from_dict, _message_to_dict
+from langchain.memory import MongoDBChatMessageHistory
+import logging
+from typing import List
+
+
+logger = logging.getLogger(__name__)
+class CustomMongoDBChatMessageHistory(MongoDBChatMessageHistory):
+
+    @property
+    def messages(self) -> List[BaseMessage]:  # type: ignore
+        """Retrieve the messages from MongoDB"""
+        from pymongo import errors
+        cursor = None
+        try:
+            cursor = self.collection.find({"SessionId": self.session_id})
+        except errors.OperationFailure as error:
+            logger.error(error)
+
+        document_count = self.collection.count_documents({"SessionId": self.session_id})
+
+        if cursor and document_count > 0:
+            document = cursor[0]  # Get the first document with the matching session id
+            items = document["messages"]  # Get the messages array from the document
+        else:
+            items = []
+
+        messages = messages_from_dict([json.loads(item) for item in items])
+        return messages
+
+    def add_user_message(self, message: str) -> None:
+        self.append(HumanMessage(content=message))
+
+    def add_ai_message(self, message: str) -> None:
+        self.append(AIMessage(content=message))
+
+    def append(self, message: BaseMessage) -> None:
+        """Append the message to the record in MongoDB with the desired format"""
+
+        # Determine the sender based on the message type
+        sender = "ai" if isinstance(message, AIMessage) else "human"
+
+
+        # Create the message object with the desired format
+        message_obj = {
+            "type": sender,
+            "content": message.content,
+            "timestamp": datetime.utcnow()
+        }
+
+        try:
+            # Update the messages array with the new message object
+            self.collection.update_one(
+                {"SessionId": self.session_id},
+                {"$push": {"messages": json.dumps(_message_to_dict(message))}},
+                upsert=True
+            )
+        except errors.WriteError as err:
+            logger.error(err)

requirements.txt

@@ -7,4 +7,5 @@ transformers
 gtts
 torch
 tiktoken
-huggingface-hub
+huggingface-hub
+pymongo

utils.py

@@ -17,143 +17,26 @@ from langchain.prompts.chat import (
 from langchain.text_splitter import CharacterTextSplitter
 from langchain.vectorstores.faiss import FAISS
 from langchain.cache import InMemoryCache
-
-# Create a dict of filenames and their corresponding url links to be used as source in line #203
-file_url_mapping = {
-    'docs/01_course-orientation/01_about-the-course/01_introduction-to-the-3d-printing-specialization.en.txt':
-        'https://www.coursera.org/learn/3d-printing-revolution/lecture/mLvWU/introduction-to-the-3d-printing-specialization',
-    'docs/01_course-orientation/01_about-the-course/02_welcome-to-the-3d-printing-revolution.en.txt':
-        'https://www.coursera.org/learn/3d-printing-revolution/lecture/zQ7lG/welcome-to-the-3d-printing-revolution',
-
-    'docs/03_module-2-why-is-it-revolutionary/03_the-3d-printing-revolution-facts-concepts/02_how-will-3d-printing-change-business.en.txt':
-        'https://www.coursera.org/learn/3d-printing-revolution/lecture/D8VUj/how-will-3d-printing-change-business',
-
-    'docs/03_module-2-why-is-it-revolutionary/03_the-3d-printing-revolution-facts-concepts/01_whats-special-about-3d-printing.en.txt':
-        'https://www.coursera.org/learn/3d-printing-revolution/lecture/WmJQL/whats-special-about-3d-printing',
-
-    'docs/03_module-2-why-is-it-revolutionary/03_the-3d-printing-revolution-facts-concepts/04_remixing-products-exercise-overview.en.txt':
-        'https://www.coursera.org/learn/3d-printing-revolution/lecture/J3Tq3/remixing-products-exercise-overview',
-
-    'docs/03_module-2-why-is-it-revolutionary/03_the-3d-printing-revolution-facts-concepts/03_the-future-of-3d-printing.en.txt':
-        'https://www.coursera.org/learn/3d-printing-revolution/lecture/tjSlh/the-future-of-3d-printing',
-
-    'docs/03_module-2-why-is-it-revolutionary/02_an-early-look-at-the-coming-revolution/01_tour-of-the-illinois-makerlab-vishal-sachdev.en.txt':
-        'https://www.coursera.org/learn/3d-printing-revolution/lecture/rcVnN/tour-of-the-illinois-makerlab-vishal-sachdev',
-
-    'docs/03_module-2-why-is-it-revolutionary/02_an-early-look-at-the-coming-revolution/02_meet-the-makers-arielle-rausin-cameron-alberg-scott-zelman.en.txt':
-        'https://www.coursera.org/learn/3d-printing-revolution/lecture/J2nD2/meet-the-makers-arielle-rausin-cameron-alberg-scott-zelman',
-    'docs/03_module-2-why-is-it-revolutionary/04_the-revolutionaries/03_shapeways-lauren-slowik.en.txt':
-        'https://www.coursera.org/learn/3d-printing-revolution/lecture/mi1zr/shapeways-lauren-slowik',
-    'docs/03_module-2-why-is-it-revolutionary/04_the-revolutionaries/04_3d-fashion-francis-bitonti.en.txt':
-        'https://www.coursera.org/learn/3d-printing-revolution/lecture/KmYxf/3d-fashion-francis-bitonti',
-    'docs/03_module-2-why-is-it-revolutionary/04_the-revolutionaries/06_whats-next-hod-lipson.en.txt':
-        'https://www.coursera.org/learn/3d-printing-revolution/lecture/3zKRN/whats-next-hod-lipson',
-    'docs/03_module-2-why-is-it-revolutionary/04_the-revolutionaries/05_3d-printed-battery-paul-braun.en.txt':
-        'https://www.coursera.org/learn/3d-printing-revolution/lecture/mrEAK/3d-printed-battery-paul-braun',
-    'docs/03_module-2-why-is-it-revolutionary/04_the-revolutionaries/02_normal-nikki-kaufman.en.txt':
-        'https://www.coursera.org/learn/3d-printing-revolution/lecture/I8QXq/normal-nikki-kaufman',
-    'docs/03_module-2-why-is-it-revolutionary/04_the-revolutionaries/01_voodoo-manufacturing-max-friefeld.en.txt':
-        'https://www.coursera.org/learn/3d-printing-revolution/lecture/atrTP/voodoo-manufacturing-max-friefeld',
-    'docs/04_course-conclusion/01_course-wrap-up-whats-next/01_the-3d-printing-revolution-wrap-up.en.txt':
-        'https://www.coursera.org/learn/3d-printing-revolution/lecture/f6XvX/the-3d-printing-revolution-wrap-up',
-    'docs/04_course-conclusion/01_course-wrap-up-whats-next/03_gies-online-programs.en.txt':
-        'https://www.coursera.org/learn/3d-printing-revolution/lecture/JdvZk/gies-online-programs',
-
-    'docs/02_module-1-what-is-3d-printing/05_optional-content/01_like-this-course-learn-more-with-the-imba-optional.en.txt':
-        'https://www.coursera.org/learn/3d-printing-revolution/lecture/MIATt/like-this-course-learn-more-with-the-imba-optional',
-    'docs/02_module-1-what-is-3d-printing/03_3d-printing-facts-concepts/04_where-designs-come-from.en.txt':
-        'https://www.coursera.org/learn/3d-printing-revolution/lecture/0fU1x/where-designs-come-from',
-    'docs/02_module-1-what-is-3d-printing/03_3d-printing-facts-concepts/05_where-to-find-3d-printers.en.txt':
-        'https://www.coursera.org/learn/3d-printing-revolution/lecture/3XTPj/where-to-find-3d-printers',
-    'docs/02_module-1-what-is-3d-printing/03_3d-printing-facts-concepts/01_history-of-3d-printing.en.txt':
-        'https://www.coursera.org/learn/3d-printing-revolution/lecture/HMvuh/history-of-3d-printing',
-    'docs/02_module-1-what-is-3d-printing/03_3d-printing-facts-concepts/02_how-3d-printers-work.en.txt':
-        'https://www.coursera.org/learn/3d-printing-revolution/lecture/dNpvw/how-3d-printers-work',
-    'docs/02_module-1-what-is-3d-printing/03_3d-printing-facts-concepts/03_materials-costs.en.txt':
-        'https://www.coursera.org/learn/3d-printing-revolution/lecture/j63D2/materials-costs',
-    'docs/02_module-1-what-is-3d-printing/03_3d-printing-facts-concepts/06_3d-printing-applications.en.txt':
-        'https://www.coursera.org/learn/3d-printing-revolution/lecture/EJdi9/3d-printing-applications',
-
-    'docs/02_module-1-what-is-3d-printing/04_more-3d-printing-insights/03_selective-laser-melting-rodrigo-gutierrez.en.txt':
-        'https://www.coursera.org/learn/3d-printing-revolution/lecture/kkpwz/selective-laser-melting-rodrigo-gutierrez',
-    'docs/02_module-1-what-is-3d-printing/04_more-3d-printing-insights/04_3d-printing-ecosystem-aaron-roy.en.txt':
-        'https://www.coursera.org/learn/3d-printing-revolution/lecture/2zjzy/3d-printing-ecosystem-aaron-roy',
-
-    'docs/02_module-1-what-is-3d-printing/04_more-3d-printing-insights/01_3d-printing-demonstration-danny-lohan.en.txt':
-        'https://www.coursera.org/learn/3d-printing-revolution/lecture/5b8IG/3d-printing-demonstration-danny-lohan',
-
-    'docs/02_module-1-what-is-3d-printing/04_more-3d-printing-insights/02_3d-printing-vs-additive-manufacturing-mark-cotteleer.en.txt':
-        'https://www.coursera.org/learn/3d-printing-revolution/lecture/qBvTb/3d-printing-vs-additive-manufacturing-mark-cotteleer',
-    'docs/02_module-1-what-is-3d-printing/02_3d-printing-insights/02_my-3d-printing-story-aric-rindfleisch.en.txt':
-        'https://www.coursera.org/learn/3d-printing-revolution/lecture/hhWc8/my-3d-printing-story-aric-rindfleisch',
-
-    'docs/02_module-1-what-is-3d-printing/02_3d-printing-insights/04_3d-printing-the-maker-movement-hackerspaces-chris-meyer.en.txt':
-        'https://www.coursera.org/learn/3d-printing-revolution/lecture/8HO88/3d-printing-the-maker-movement-hackerspaces-chris-meyer',
-    'docs/02_module-1-what-is-3d-printing/02_3d-printing-insights/06_what-would-you-make-exercise-overview.en.txt':
-        'https://www.coursera.org/learn/3d-printing-revolution/lecture/kfDp2/what-would-you-make-exercise-overview',
-
-    'docs/02_module-1-what-is-3d-printing/02_3d-printing-insights/05_the-birth-of-desktop-3d-printing-matt-griffin.en.txt':
-        'https://www.coursera.org/learn/3d-printing-revolution/lecture/RlvEF/the-birth-of-desktop-3d-printing-matt-griffin',
-    'docs/02_module-1-what-is-3d-printing/02_3d-printing-insights/03_tour-of-sector67-chris-meyer.en.txt':
-        'https://www.coursera.org/learn/3d-printing-revolution/lecture/HXJrm/tour-of-sector67-chris-meyer',
-    'docs/02_module-1-what-is-3d-printing/02_3d-printing-insights/01_views-on-3d-printing-champaign-new-york.en.txt':
-        'https://www.coursera.org/learn/3d-printing-revolution/lecture/jvULv/views-on-3d-printing-champaign-new-york',
-    'docs/01_course-orientation/01_about-the-course/05_glossary_instructions.html':
-        'https://www.coursera.org/learn/3d-printing-revolution/supplement/515pX/glossary',
-    'docs/01_course-orientation/01_about-the-course/03_syllabus_instructions.html':
-        'https://www.coursera.org/learn/3d-printing-revolution/supplement/l2J8d/syllabus',
-    'docs/01_course-orientation/01_about-the-course/06_learner-stories_instructions.html':
-        'https://www.coursera.org/learn/3d-printing-revolution/supplement/goE8a/learner-stories',
-    'docs/01_course-orientation/01_about-the-course/04_about-the-discussion-forums_instructions.html':
-        'https://www.coursera.org/learn/3d-printing-revolution/supplement/HA7H7/about-the-discussion-forums',
-    'docs/01_course-orientation/02_about-your-classmates/02_social-media_illinois.edu.html':
-        'https://www.facebook.com/illinois.edu',
-    'docs/01_course-orientation/02_about-your-classmates/01_updating-your-profile_instructions.html':
-        'https://www.coursera.org/learn/3d-printing-revolution/supplement/f2iFO/updating-your-profile',
-
-    'docs/01_course-orientation/02_about-your-classmates/02_social-media_termsDknlfPe-5xKJJaHWl4j6kveY2GjKiZxK_vd-JDIN3-6piB5w.2PhI_l8TrILDNPPI-v-VPg.kolgdmfhbhzcbgm3-lv_.html':
-        'https://www.coursera.org/about/termsDknlfPe-5xKJJaHWl4j6kveY2GjKiZxK_vd-JDIN3-6piB5w.2PhI_l8TrILDNPPI-v-VPg.kOLGdmfhbhzCbGM3-lv_j477mwb9Ffuxi5xhEac6Biqu1NvspELKPGjNRjoAzuv8LasLtq22lxzrgdd9C8Y4JQ4gHm7FuRqTL4rlby3Pb_N4mpVXMkT83a3Ob_0QWVHv7LiZghGDTwCWYxU4lMfZpfsqsm7PoQ7HfKUFvHwgUIbOGox3ZZgJtBE2t-TDkbegcktpcn6k2VqZZ0WYvQTad7oijs5WHJLfL7EYiUGb01udFqMaOLIPP1msztyo496GDNUgBSvsJcPfHE20dluqe5_KzaSoXzKxXdiW12DjKJk_XDNc14mf41U17h5HgMXg',
-    'docs/01_course-orientation/02_about-your-classmates/02_social-media_instructions.html':
-        'https://www.coursera.org/learn/3d-printing-revolution/supplement/1vga3/social-media',
-
-    'docs/03_module-2-why-is-it-revolutionary/03_the-3d-printing-revolution-facts-concepts/05_remixing-products-exercise_peer_assignment_instructions.html':
-        'https://www.coursera.org/learn/3d-printing-revolution/peer/FAZ0P/remixing-products-exercise',
-    'docs/03_module-2-why-is-it-revolutionary/01_module-2-information/01_module-2-overview_instructions.html':
-        'https://www.coursera.org/learn/3d-printing-revolution/supplement/UwuNp/module-2-overview',
-    'docs/04_course-conclusion/01_course-wrap-up-whats-next/02_congratulations_instructions.html':
-        'https://www.coursera.org/learn/3d-printing-revolution/supplement/vISgA/congratulations',
-    'docs/05_Resources/03_3d-printing-services-and-products/01__resources.html':
-        'https://www.coursera.org/learn/3d-printing-revolution/resources/aPz3d',
-    'docs/05_Resources/04_3d-printing-community/01__resources.html':
-        'https://www.coursera.org/learn/3d-printing-revolution/resources/lfbM9',
-    'docs/05_Resources/02_3d-printing-softwares/01__resources.html':
-        'https://www.coursera.org/learn/3d-printing-revolution/resources/ltlHt',
-    'docs/05_Resources/01_books-articles/01__resources.html':
-        'https://www.coursera.org/learn/3d-printing-revolution/resources/FH3x3',
-    'docs/05_Resources/05_explore-the-imba/01__resources.html':
-        'https://www.coursera.org/learn/3d-printing-revolution/resources/0AejF',
-    'docs/02_module-1-what-is-3d-printing/01_module-1-overview/01_module-1-overview_instructions.html':
-        'https://www.coursera.org/learn/3d-printing-revolution/supplement/HZXB5/module-1-overview',
-
-    'docs/02_module-1-what-is-3d-printing/02_3d-printing-insights/07_what-would-you-make-exercise_peer_assignment_instructions.html':
-        'https://www.coursera.org/learn/3d-printing-revolution/peer/t8bqq/what-would-you-make-exercise'}
-
+from memory import CustomMongoDBChatMessageHistory
+from mapping import FILE_URL_MAPPING
 
 langchain.llm_cache = InMemoryCache()
 
-global model_name
-
 models = ["GPT-3.5", "Flan UL2", "GPT-4", "Flan T5"]
 
 pickle_file = "_vs.pkl"
 index_file = "_vs.index"
 models_folder = "models/"
+MONGO_DB_URL = os.environ['MONGO_DB_URL']
 
 llm = ChatOpenAI(model_name="gpt-4", temperature=0.1)
 
 embeddings = OpenAIEmbeddings(model='text-embedding-ada-002')
 
-chat_history = []
+message_history = CustomMongoDBChatMessageHistory(
+    connection_string=MONGO_DB_URL, session_id='session_id', database_name='coursera_bots',
+    collection_name='3d_printing_revolution'
+)
 
 memory = ConversationBufferWindowMemory(memory_key="chat_history", k=10)
 
@@ -172,11 +55,24 @@ messages = [
 CHAT_PROMPT = ChatPromptTemplate.from_messages(messages)
 
 
+def set_session_id(session_id):
+    global message_history, memory
+    # check if message_history with same session id exists
+    if message_history.session_id == session_id:
+        print("Session id already set: " + str(message_history.session_id))
+    else:
+        # create new message history with session id
+        print("Setting session id to " + str(session_id))
+        message_history = CustomMongoDBChatMessageHistory(
+            connection_string=MONGO_DB_URL, session_id=session_id, database_name='coursera_bots',
+            collection_name='printing_3d_revolution'
+        )
+        memory = ConversationBufferWindowMemory(memory_key="chat_history", chat_memory=message_history, k=10, return_messages=True)
+
+
 def set_model_and_embeddings(model):
-    global chat_history
     set_model(model)
     # set_embeddings(model)
-    chat_history = []
 
 
 def set_model(model):
@@ -263,7 +159,7 @@ def fetch_data_for_embeddings():
 
     # use file_url_mapping to set metadata of document to url which has been set as the source
     for document in document_list:
-        document.metadata["url"] = file_url_mapping.get(document.metadata["source"])
+        document.metadata["url"] = FILE_URL_MAPPING.get(document.metadata["source"])
 
     print("document list: " + str(len(document_list)))
     return document_list
@@ -288,7 +184,7 @@ def create_chunk_documents():
 
 
 def get_qa_chain(vectorstore_index):
-    global llm, model_name
+    global llm
     print(llm)
 
     # embeddings_filter = EmbeddingsFilter(embeddings=embeddings, similarity_threshold=0.76)
@@ -297,7 +193,7 @@ def get_qa_chain(vectorstore_index):
                                                search_kwargs={"score_threshold": .7})
 
     chain = ConversationalRetrievalChain.from_llm(llm, retriever, return_source_documents=True,
-                                                  verbose=True, get_chat_history=get_chat_history,
+                                                  verbose=True,
                                                   combine_docs_chain_kwargs={"prompt": CHAT_PROMPT})
     return chain
 
@@ -310,12 +206,17 @@ def get_chat_history(inputs) -> str:
 
 
 def generate_answer(question) -> str:
-    global chat_history, vectorstore_index
+    global vectorstore_index
     chain = get_qa_chain(vectorstore_index)
-
+    history = memory.chat_memory.messages
+    print("chat history: " + str(history))
     result = chain(
-        {"question": question, "chat_history": chat_history, "vectordbkwargs": {"search_distance": 0.6}})
-    chat_history = [(question, result["answer"])]
+        {"question": question, "chat_history": history})
+    # chat_tuple = (question, result["answer"])
+    # chat_history.append(chat_tuple)
+    save_chat_history(question, result)
+    chat_history = [message.content for message in memory.chat_memory.messages]
+    print("chat history after response: " + str(chat_history))
     sources = []
     print(result)
 
@@ -326,3 +227,10 @@ def generate_answer(question) -> str:
 
     source = ',\n'.join(set(sources))
     return result['answer'] + '\nSOURCES: ' + source
+
+
+def save_chat_history(question, result):
+
+    memory.chat_memory.add_user_message(question)
+    memory.chat_memory.add_ai_message(result["answer"])
+    print("chat history on saving: " + str(memory.chat_memory.messages))