initial push

.env.example

@@ -0,0 +1 @@
+OPENAI_API_KEY=

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+data/index.faiss filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,10 @@
+__pycache__/
+.vscode/
+.chainlit/
+
+.env
+env/
+venv/
+
+pdf_data*
+reg_gpt_*

app.py

@@ -0,0 +1,170 @@
+import os
+import openai
+import gradio as gr
+from dotenv import load_dotenv
+from utils import chat
+from config import CFG_APP
+
+# Load API KEY
+try:
+    load_dotenv()
+except Exception as e:
+    pass
+openai.api_key = os.environ["OPENAI_API_KEY"]
+
+
+# SYS Template
+system_template = {
+    "role": "system",
+    "content": CFG_APP.INIT_PROMPT,
+}
+
+# APP
+theme = gr.themes.Monochrome(
+    font=[gr.themes.GoogleFont("Kanit"), "sans-serif"],
+)
+
+with gr.Blocks(title=CFG_APP.BOT_NAME, css="assets/style.css", theme=theme) as demo:
+    gr.Markdown(f"<h1><center>{CFG_APP.BOT_NAME} 🤖</center></h1>")
+
+    with gr.Row():
+        with gr.Column(scale=2):
+            chatbot = gr.Chatbot(
+                elem_id="chatbot", label=f"{CFG_APP.BOT_NAME} chatbot", show_label=False
+            )
+            state = gr.State([system_template])
+
+            with gr.Row():
+                ask = gr.Textbox(
+                    show_label=False,
+                    placeholder="Ask here your question and press enter",
+                ).style(container=False)
+                ask_examples_hidden = gr.Textbox(elem_id="hidden-message")
+
+            examples_questions = gr.Examples(
+                [*CFG_APP.DEFAULT_QUESTIONS],
+                [ask_examples_hidden],
+                examples_per_page=15,
+            )
+
+        with gr.Column(scale=1, variant="panel"):
+            gr.Markdown("### Sources")
+            sources_textbox = gr.Markdown(show_label=False)
+
+    ask.submit(
+        fn=chat,
+        inputs=[ask, state],
+        outputs=[chatbot, state, sources_textbox],
+    )
+    ask.submit(lambda x: gr.update(value=""), [], [ask])
+
+    ask_examples_hidden.change(
+        fn=chat,
+        inputs=[ask_examples_hidden, state],
+        outputs=[chatbot, state, sources_textbox],
+    )
+    demo.queue(concurrency_count=16)
+    gr.Markdown(
+        """
+    
+    ### 🎯 Understanding ESMA-GPT's Purpose
+    
+    In a context where financial regulations are constantly evolving, direct access to accurate information requires significant effort in sorting through regulatory documents. Given the fluidity of these regulations and the sheer volume of legislative paperwork, pinpointing precise, actionable information can be a daunting task. 
+
+
+    \n Reg-GPT, a conversational tool related to a chatbot, offers an effective solution to this challenge. ESMA-GPT is specifically designed to address queries related to credit risk regulations. This tool draws its insights solely from documents published by official European regulatory sources, thus assuring the reliability and pertinence of its responses. By strictly focusing on these documents, ESMA-GPT ensures that it does not reference non-relevant sources, maintaining a high standard of precision in its responses. This novel tool harnesses the power of conversational AI to help users navigate the complex world of credit risk regulations, simplifying the task and promoting compliance efficiency.
+    
+    """
+    )
+
+    gr.Markdown(
+        """
+    
+    ### 📃 Inputs and functionalities
+    
+    In its initial release, Version 0, ESMA-GPT uses the subsequent 13 documents as the basis for its answers:
+    \n
+    |Document|Link|
+    |:----|:----|
+    |UCITS|https://eur-lex.europa.eu/legal-content/EN/TXT/?uri=CELEX%3A02009L0065-20230101|
+    |AIFMD|https://eur-lex.europa.eu/legal-content/EN/TXT/?uri=CELEX%3A02011L0061-20210802&qid=1692715565602|
+    |CRAR|https://eur-lex.europa.eu/legal-content/EN/TXT/?uri=CELEX%3A02009R1060-20190101&qid=1634569841934|
+    |EMIR|https://eur-lex.europa.eu/legal-content/EN/TXT/?uri=CELEX%3A02012R0648-20220812|
+    |Benchmarks Regulation|https://eur-lex.europa.eu/legal-content/EN/TXT/?uri=CELEX%3A02016R1011-20220101|
+    |MIFIR|https://eur-lex.europa.eu/legal-content/EN/TXT/?uri=CELEX%3A02014R0600-20220101|
+    |MIFID II|https://eur-lex.europa.eu/legal-content/EN/TXT/?uri=CELEX%3A02014L0065-20230323|
+    |CSDR|https://eur-lex.europa.eu/legal-content/EN/TXT/?uri=CELEX%3A02014R0909-20220622|
+    |SFTR|https://eur-lex.europa.eu/legal-content/EN/TXT/?uri=CELEX%3A02015R2365-20220812|
+    |Prospectus Regulation|https://eur-lex.europa.eu/legal-content/EN/TXT/?uri=CELEX%3A02017R1129-20211110|
+    |SSR|https://eur-lex.europa.eu/legal-content/EN/TXT/?uri=CELEX%3A02012R0236-20220131|
+    |SECR|https://eur-lex.europa.eu/legal-content/EN/TXT/?uri=CELEX%3A02017R2402-20210409|
+    |Transparency Directive|https://eur-lex.europa.eu/legal-content/EN/TXT/?uri=CELEX%3A02017R1129-20211110|
+    """
+    )
+
+    gr.Markdown(
+        """
+
+    ESMA-GPT provides users with the opportunity to input queries using a dedicated prompt area, much like the one used in OpenAI's ChatGPT. If you're unsure of what to ask, examples of potential questions are displayed below the query bar. Simply click on one of these and the tool will generate corresponding responses.
+
+
+    \n When a query is submitted to the model, 10 sources are extracted from the previously mentioned documents to provide a comprehensive answer. These sources are quoted within the generated answer to ensure accuracy and reliability. For easy reference, exact passages can be quickly located by clicking on the link icon 🔗 located beneath each excerpt, which will directly guide you to the relevant section within the document.
+    
+    """
+    )
+
+    gr.Markdown(
+        """
+    
+    ### 💬 Prompt Initialization
+    
+    To limit the model's responses to only the 10 proposed sources, a set of prompts has been designed and will serve as instructions to the GPT API. This design decision ensures that the model's output is reliably grounded in the selected documents, contributing to the overall accuracy and reliability of the tool. The structured guidance provided by these prompts enables the GPT API to more effectively navigate the wealth of information contained within the ten sources, delivering highly relevant and concise responses to the users' queries.
+
+    <u>Prompts used to initialize ESMA-GPT: </u>
+
+    - "You are ESMA-GPT, an expert in market regulations, an AI Assistant by Nexialog Consulting."
+    - "You are given a question and extracted parts of regulation reports."
+    - "Provide a clear and structured answer based only on the context provided."
+    - "When relevant, use bullet points and lists to structure your answers."
+    - "When relevant, use facts and numbers from the following documents in your answer."
+    - "Whenever you use information from a document, reference it at the end of the sentence (ex: [doc 2])."
+    - "You don't have to use all documents, only if it makes sense in the conversation."
+    - "Don't make up new sources and references that don't exist."
+    - "If no relevant information to answer the question is present in the documents, just say you don't have enough information to answer."
+
+
+    
+    """
+    )
+
+    gr.Markdown(
+        """
+    
+    ### ⚙️Technical features
+    
+    ESMA-GPT operates through two core modules, the GPT API from OpenAI and an embedding model. The functioning of these components is integrated into a seamless workflow, which can be summarized in the figure below :
+
+
+    <div style="display:flex; justify-content:center;">
+    <img src="file/Images/Reg-GPT.png" width="800" height="800" />
+    </div>
+ 
+
+    - Open AI Api version :  gpt-3.5-turbo
+    - Embedding model : https://huggingface.co/sentence-transformers/multi-qa-mpnet-base-dot-v1
+
+
+
+    
+    """
+    )
+    gr.Markdown(
+        "<h1><center>Disclaimer ⚠️</center></h1>\n"
+        + """
+- Please be aware that this is Version 0 of our application. You may encounter certain errors or glitches as we continue to refine and enhance its functionality. You might experience some nonsensical answers, similar to those experienced when using chat-GPT. If you encounter any issues, don't hesitate to reach out to us at [email protected].
+- Our application relies on an external API provided by OpenAI. There may be instances where errors occur due to high demand on the API. If you encounter such an issue, we recommend that you refresh the page and retry your query, or try again a little bit later.
+- When using our application, we urge you to ask clear and explicit questions that adhere to the scope of financial market regulations. This will ensure that you receive the most accurate and relevant responses from the system.
+"""
+    )
+
+demo.launch()

assets/style.css

@@ -0,0 +1,184 @@
+.warning-box {
+    background-color: #fff3cd;
+    border: 1px solid #ffeeba;
+    border-radius: 4px;
+    padding: 15px 20px;
+    font-size: 14px;
+    color: #856404;
+    display: inline-block;
+    margin-bottom: 15px;
+}
+
+
+.tip-box {
+    background-color: #f0f9ff;
+    border: 1px solid #80d4fa;
+    border-radius: 4px;
+    margin-top: 20px;
+    padding: 15px 20px;
+    font-size: 14px;
+    color: #006064;
+    display: inline-block;
+    margin-bottom: 15px;
+    width: auto;
+}
+
+.tip-box-title {
+    font-weight: bold;
+    font-size: 14px;
+    margin-bottom: 5px;
+}
+
+.light-bulb {
+    display: inline;
+    margin-right: 5px;
+}
+
+.gr-box {
+    border-color: #d6c37c
+}
+
+#hidden-message {
+    display: none;
+}
+
+.message {
+    font-size: 14px !important;
+}
+
+
+a {
+    text-decoration: none;
+    color: inherit;
+}
+
+.card {
+    background-color: #233f55;
+    border-radius: 10px;
+    box-shadow: 0 4px 6px rgba(0, 0, 0, 0.1);
+    overflow: hidden;
+    display: flex;
+    flex-direction: column;
+    margin: 20px;
+}
+
+.card-content {
+    padding: 20px;
+}
+
+.card-content h2 {
+    font-size: 14px !important;
+    font-weight: bold;
+    margin-bottom: 10px;
+    margin-top: 0px !important;
+    color: #577b9b !important;
+    ;
+}
+
+.card-content p {
+    font-size: 12px;
+    margin-bottom: 0;
+}
+
+.card-footer {
+    background-color: #f4f4f4;
+    font-size: 10px;
+    padding: 10px;
+    display: flex;
+    justify-content: space-between;
+    align-items: center;
+}
+
+.card-footer span {
+    flex-grow: 1;
+    text-align: left;
+    color: #999 !important;
+}
+
+.pdf-link {
+    display: inline-flex;
+    align-items: center;
+    margin-left: auto;
+    text-decoration: none !important;
+    font-size: 14px;
+}
+
+
+
+.message.user {
+    background-color: #b20032 !important;
+    border: none;
+    color: white !important;
+}
+
+.message.bot {
+    /* background-color: #f2f2f7 !important; */
+    border: none;
+}
+
+.gallery-item>div:hover {
+    background-color: #7494b0 !important;
+    color: white !important;
+}
+
+.gallery-item:hover {
+    border: #7494b0 !important;
+}
+
+.gallery-item>div {
+    background-color: white !important;
+    color: #577b9b !important;
+}
+
+.label {
+    color: #577b9b !important;
+}
+
+.paginate {
+    color: #577b9b !important;
+}
+
+
+label>span {
+    background-color: white !important;
+    color: #577b9b !important;
+}
+
+/* Pseudo-element for the circularly cropped picture */
+.message.bot::before {
+    content: '';
+    position: absolute;
+    top: -10px;
+    left: -10px;
+    width: 30px;
+    height: 30px;
+    background-image: url('https://www.nexialog.com/wp-content/uploads/2021/10/cropped-icone-onglet-logo.png');
+    background-color: #fff;
+    background-size: cover;
+    background-position: center;
+    border-radius: 50%;
+    z-index: 10;
+}
+
+
+
+.user.svelte-6roggh.svelte-6roggh {
+    padding: 17px 24px;
+    text-align: justify;
+}
+
+.gallery.svelte-1ayixqk {
+    text-align: left;
+}
+
+.card-content p,
+.card-content ul li {
+    color: #fff !important;
+}
+
+.message.bot, .bot.svelte-6roggh.svelte-6roggh  {
+    background: #233f55 !important;
+    padding: 17px 24px !important;
+    text-align: justify !important;
+    color: #fff !important;
+}

config.py

@@ -0,0 +1,61 @@
+class CFG_APP:
+    DEBUG = True
+    K_TOTAL = 10
+    THRESHOLD = 0.3
+    DEVICE = "cpu"
+    BOT_NAME = "ESMA-GPT"
+    MODEL_NAME = "gpt-3.5-turbo"
+    DEFAULT_LANGUAGE = "English"
+
+    DATA_FOLDER = "data/"
+    EMBEDDING_MODEL = "sentence-transformers/multi-qa-mpnet-base-dot-v1"
+
+    MAX_TOKENS_REF_QUESTION = 128  # Number of tokens in reformulated question
+    MAX_TOKENS_ANSWER = 1024  # Number of tokens in answers
+    MAX_TOKENS_API = 3100
+    INIT_PROMPT = (
+        f"You are {BOT_NAME}, an expert in market regulations, an AI Assistant by Nexialog Consulting. "
+        "You are given a question and extracted parts of regulation reports."
+        "Provide a clear and structured answer based only on the context provided. "
+        "When relevant, use bullet points and lists to structure your answers."
+    )
+    SOURCES_PROMPT = (
+        "When relevant, use facts and numbers from the following documents in your answer. "
+        "Whenever you use information from a document, reference it at the end of the sentence (ex: [doc 2]). "
+        "You don't have to use all documents, only if it makes sense in the conversation. "
+        "If no relevant information to answer the question is present in the documents, "
+        "just say you don't have enough information to answer."
+    )
+
+    DEFAULT_QUESTIONS = (
+        "What is the definition of PD ?",
+        "What is the definition of LGD ?",
+        "What is the definition of EAD ?",
+        "What is the definition of EL ?",
+        "What is the definition of ELBE ?",
+        "What does the credit conversion factor correspond to?",
+        "Is it mandatory to implement a credit conversion factor?",
+        "Quand doit on appliquer la période probatoire ?",
+        "Comment doivent étre traité les mutliples defaut ?",
+        "What is the significance of Directive 2009/65/EC, also known as the UCITS Directive?",
+        "How do these directives and regulations interact and influence each other as per the passage?",
+    )
+
+    REFORMULATION_PROMPT = """
+        Important ! Give the output as a standalone question followed by the detected language whatever the form of the query.
+        Reformulate the following user message to be a short standalone question in English, in the context of an educational discussion about regulations in banks. Then detect the language of the query
+        Sometimes, explanations of some abbreviations will be given in parentheses, keep them.
+        ---
+        query: C'est quoi les régles que les banques américaines doivent suivre ?
+        standalone question: What are the key regulations that banks in the United States must follow?
+        language: French
+        ---
+        query: what are the main effects of bank regulations?
+        standalone question: What are the main causes of bank regulations change in the last century?
+        language: English
+        ---
+        query: UL (Unexpected Loss)
+        standalone question: What does UL (Unexpected Loss) stand for?
+        language: English
+    """
+    DOC_METADATA_PATH = f"{DATA_FOLDER}/doc_metadata.json"

data/cache-54e827f71a3e8391.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:56b5975c2b06abb970c3f950a56945773d519aade724c8ed6dfc121ee1da0ee4
+size 18548936

data/data-00000-of-00001.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ca0fda10fa92567385c1820b39fe16cc416cefdfc13f934a658e5aeed31d074c
+size 3555776

data/dataset_info.json

@@ -0,0 +1,52 @@
+{
+  "citation": "",
+  "description": "",
+  "features": {
+    "id": {
+      "dtype": "string",
+      "_type": "Value"
+    },
+    "document_id": {
+      "dtype": "string",
+      "_type": "Value"
+    },
+    "content_type": {
+      "dtype": "string",
+      "_type": "Value"
+    },
+    "content": {
+      "dtype": "string",
+      "_type": "Value"
+    },
+    "length": {
+      "dtype": "int64",
+      "_type": "Value"
+    },
+    "idx_block": {
+      "dtype": "int64",
+      "_type": "Value"
+    },
+    "page_number": {
+      "dtype": "int64",
+      "_type": "Value"
+    },
+    "x0": {
+      "dtype": "float64",
+      "_type": "Value"
+    },
+    "y0": {
+      "dtype": "float64",
+      "_type": "Value"
+    },
+    "x1": {
+      "dtype": "float64",
+      "_type": "Value"
+    },
+    "y1": {
+      "dtype": "float64",
+      "_type": "Value"
+    }
+  },
+  "homepage": "",
+  "license": ""
+}

data/doc_metadata.json

@@ -0,0 +1 @@
+[{"id": "bc5f7057b9f4e43d6e8caf48b6fc3356", "title": "CL2012R0236EN0020010.0001_cp 1..1", "author": "Publications Office", "subject": " ", "creation_date": "D:20220201194303+05'00'", "modification_date": "D:20220209155755+01'00'", "n_pages": 35, "url": "/content/pdf_data/REGULATION_236-2012-EU.pdf", "file_name": "REGULATION_236-2012-EU.pdf", "short_name": "REGULATION_236-2012-EU.pdf", "release_date": "", "report_type": "", "source": ""}, {"id": "820d40403125900ccb2b79944064c4a2", "title": "CL2012R0648EN0200010.0001.3bi_cp 1..1", "author": "Publications Office", "subject": " ", "creation_date": "D:20220811063549-07'00'", "modification_date": "D:20220909063727+02'00'", "n_pages": 149, "url": "/content/pdf_data/REGULATION_648-2012-EU.pdf", "file_name": "REGULATION_648-2012-EU.pdf", "short_name": "REGULATION_648-2012-EU.pdf", "release_date": "", "report_type": "", "source": ""}, {"id": "c3e2edf8845964848e971f6636511c24", "title": "CL2014L0065EN0100010.0001_cp 1..1", "author": "Publications Office", "subject": " ", "creation_date": "D:20230425120739+05'00'", "modification_date": "D:20230427131933+02'00'", "n_pages": 137, "url": "/content/pdf_data/DIRECTIVE_2014-65-EU.pdf", "file_name": "DIRECTIVE_2014-65-EU.pdf", "short_name": "DIRECTIVE_2014-65-EU.pdf", "release_date": "", "report_type": "", "source": ""}, {"id": "6d8ed9ae4db9ecdc6cebfb06dd4f2f0b", "title": "CL2014R0909EN0020010.0001_cp 1..1", "author": "Publications Office", "subject": " ", "creation_date": "D:20220624104619+05'00'", "modification_date": "D:20220707101313+02'00'", "n_pages": 72, "url": "/content/pdf_data/REGULATION_909-2014-EU.pdf", "file_name": "REGULATION_909-2014-EU.pdf", "short_name": "REGULATION_909-2014-EU.pdf", "release_date": "", "report_type": "", "source": ""}, {"id": "ddde285e11098c904e134ba55ed7bd4d", "title": "CL2009R1060EN0060010.0001.3bi_cp 1..1", "author": "Publications Office", "subject": " ", "creation_date": "D:20190204193440+05'00'", "modification_date": "D:20190227072321+01'00'", "n_pages": 79, "url": "/content/pdf_data/REGULATION_1060-2009-EC.pdf", "file_name": "REGULATION_1060-2009-EC.pdf", "short_name": "REGULATION_1060-2009-EC.pdf", "release_date": "", "report_type": "", "source": ""}, {"id": "d72d80bb0701ac884dabf10cdc6dc53e", "title": "CL2009L0065EN0090010.0001_cp 1..1", "author": "Publications Office", "subject": " ", "creation_date": "D:20230113114146+05'00'", "modification_date": "D:20230119111742+01'00'", "n_pages": 122, "url": "/content/pdf_data/DIRECTIVE_2009-65-EC.pdf", "file_name": "DIRECTIVE_2009-65-EC.pdf", "short_name": "DIRECTIVE_2009-65-EC.pdf", "release_date": "", "report_type": "", "source": ""}, {"id": "5b17e8bf36ccf6c247667effeb65399e", "title": "CL2015R2365EN0030010.0001_cp 1..1", "author": "Publications Office", "subject": " ", "creation_date": "D:20220826162622+05'00'", "modification_date": "D:20220908084959+02'00'", "n_pages": 35, "url": "/content/pdf_data/REGULATION_2015-2365-EU.pdf", "file_name": "REGULATION_2015-2365-EU.pdf", "short_name": "REGULATION_2015-2365-EU.pdf", "release_date": "", "report_type": "", "source": ""}, {"id": "8539ec20f98b637884edb4834d0dea8e", "title": "CL2016R1011EN0030010.0001.3bi_cp 1..1", "author": "Publications Office", "subject": " ", "creation_date": "D:20211125082927-08'00'", "modification_date": "D:20211208134405+01'00'", "n_pages": 96, "url": "/content/pdf_data/REGULATION_2016-1011-EU.pdf", "file_name": "REGULATION_2016-1011-EU.pdf", "short_name": "REGULATION_2016-1011-EU.pdf", "release_date": "", "report_type": "", "source": ""}, {"id": "700e9a1ef65bfdb31a791fcf5cedf2ec", "title": "CL2004L0109EN0050010.0001_cp 1..1", "author": "Publications Office", "subject": " ", "creation_date": "D:20230110134218+05'00'", "modification_date": "D:20230110160657+01'00'", "n_pages": 43, "url": "/content/pdf_data/DIRECTIVE_2004-109-EC.pdf", "file_name": "DIRECTIVE_2004-109-EC.pdf", "short_name": "DIRECTIVE_2004-109-EC.pdf", "release_date": "", "report_type": "", "source": ""}, {"id": "8d385be7c329beb580517ebc0014c010", "title": "CL2014R0600EN0050030.0001.3bi_cp 1..1", "author": "Publications Office", "subject": " ", "creation_date": "D:20220629042951-07'00'", "modification_date": "D:20220809164129+02'00'", "n_pages": 102, "url": "/content/pdf_data/REGULATION_600-2014-EU.pdf", "file_name": "REGULATION_600-2014-EU.pdf", "short_name": "REGULATION_600-2014-EU.pdf", "release_date": "", "report_type": "", "source": ""}, {"id": "ae33cf33064c53752fec4d4f9cc3d67e", "title": "CL2011L0061EN0050010.0001_cp 1..1", "author": "Publications Office", "subject": " ", "creation_date": "D:20210812120133+05'00'", "modification_date": "D:20210818032253+02'00'", "n_pages": 105, "url": "/content/pdf_data/DIRECTIVE_2011-61-EU.pdf", "file_name": "DIRECTIVE_2011-61-EU.pdf", "short_name": "DIRECTIVE_2011-61-EU.pdf", "release_date": "", "report_type": "", "source": ""}, {"id": "8bf24201599f4be3cf071f3eade7287a", "title": "CL2017R2402EN0010010.0001_cp 1..1", "author": "Publications Office", "subject": " ", "creation_date": "D:20210527084107+05'00'", "modification_date": "D:20210527145318+02'00'", "n_pages": 76, "url": "/content/pdf_data/REGULATION_2017-2402-EU.pdf", "file_name": "REGULATION_2017-2402-EU.pdf", "short_name": "REGULATION_2017-2402-EU.pdf", "release_date": "", "report_type": "", "source": ""}, {"id": "f47a78d3fb33297ff04a9fd72608864d", "title": "CL2017R1129EN0030010.0001.3bi_cp 1..1", "author": "Publications Office", "subject": " ", "creation_date": "D:20211018181748+05'00'", "modification_date": "D:20211020184313+02'00'", "n_pages": 89, "url": "/content/pdf_data/REGULATION_2017-1129-EU.pdf", "file_name": "REGULATION_2017-1129-EU.pdf", "short_name": "REGULATION_2017-1129-EU.pdf", "release_date": "", "report_type": "", "source": ""}]

data/index.faiss

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ef8aac94cb99563186e4f813b86e2fe9c62015ee6f575cbe64b60d614147b59c
+size 14972973

data/state.json

@@ -0,0 +1,13 @@
+{
+  "_data_files": [
+    {
+      "filename": "data-00000-of-00001.arrow"
+    }
+  ],
+  "_fingerprint": "9c06f1195d938894",
+  "_format_columns": null,
+  "_format_kwargs": {},
+  "_format_type": null,
+  "_output_all_columns": false,
+  "_split": null
+}

glossary.json

@@ -0,0 +1,287 @@
+{
+    "ABoR": "Administrative Board of Review",
+    "ABS": "asset-backed security",
+    "ABSPP": "asset-backed securities purchase programme",
+    "ACH": "automated clearing house",
+    "AIF": "alternative investment fund",
+    "AMA": "advanced measurement approach",
+    "AMC": "asset management company",
+    "AMI-Pay": "Advisory Group on Market Infrastructures for Payments",
+    "AMI-SeCo": "Advisory Group on Market Infrastructures for Securities and Collateral",
+    "AML": "anti-money laundering",
+    "API": "application programming interface",
+    "APP": "asset purchase programme",
+    "ASC": "Advisory Scientific Committee",
+    "ASLP": "automated security lending programme",
+    "AT1": "Additional Tier 1",
+    "ATC": "Advisory Technical Committee",
+    "ATM": "automated teller machine",
+    "b.o.p.": "balance of payments",
+    "BCBS": "Basel Committee on Banking Supervision",
+    "BCPs": "Basel Core Principles",
+    "BEPGs": "Broad Economic Policy Guidelines",
+    "BIC": "Business Identifier Code",
+    "BIS": "Bank for International Settlements",
+    "BPM6": "Balance of Payments and International Investment Position Manual",
+    "bps": "basis points",
+    "BRM": "breach reporting mechanism",
+    "BRRD": "Bank Recovery and Resolution Directive",
+    "c.i.f.": "Cost, insurance and freight at the importer’s border",
+    "CAPE": "cyclically adjusted price/earnings (ratio)",
+    "CAPM": "capital asset pricing model",
+    "CAS": "capital adequacy statement",
+    "CBOE": "Chicago Board Options Exchange",
+    "CBPP": "covered bond purchase programme",
+    "CBR": "combined buffer requirement",
+    "CCBM": "correspondent central banking model",
+    "CCBM2": "Collateral Central Bank Management",
+    "CCoB": "capital conservation buffer",
+    "CCP": "central counterparty",
+    "CCyB": "countercyclical capital buffer",
+    "CDS": "credit default swap",
+    "CESR": "Committee of European Securities Regulators",
+    "CET1": "Common Equity Tier 1",
+    "CGFS": "Committee on the Global Financial System",
+    "CGO": "Compliance and Governance Office",
+    "CISS": "composite indicator of systemic stress",
+    "CJEU": "Court of Justice of the European Union",
+    "CMU": "capital markets union",
+    "CO2": "carbon dioxide",
+    "COGESI": "Contact Group on Euro Securities Infrastructures",
+    "COI": "Centralised On-Site Inspections Division",
+    "COREP": "common reporting",
+    "CPI": "consumer price index",
+    "CPMI": "Committee on Payments and Market Infrastructures",
+    "CPSIPS": "Core Principles for Systemically Important Payment Systems",
+    "CRD": "Capital Requirements Directive",
+    "CRE": "commercial real estate",
+    "CRR": "Capital Requirements Regulation",
+    "CSD": "central securities depository ",
+    "CSPP": "corporate sector purchase programme",
+    "D-SIB": "domestic systemically important bank",
+    "DFR": "deposit facility rate",
+    "DG ECFIN": "Directorate General for Economic and Financial Affairs, European Commission",
+    "DGS": "deposit guarantee scheme",
+    "DLT": "distributed ledger technology",
+    "DSR": "debt service ratio",
+    "DSTI": "debt service-to-income",
+    "DTA": "deferred tax asset",
+    "DTI": "debt-to-income",
+    "DvD": "delivery versus delivery",
+    "DvP": "delivery versus payment",
+    "EAD": "exposure at default",
+    "EBA": "European Banking Authority",
+    "EBITDA": "earnings before interest, taxes, depreciation and amortisation",
+    "EBP": "excess bond premium",
+    "EBPP": "Electronic Bill Presentment and Payment ",
+    "ECA": "European Court of Auditors",
+    "ECAF": "Eurosystem credit assessment framework",
+    "ECB": "European Central Bank ",
+    "ECL": "expected credit loss",
+    "ECOFIN": "Economic and Financial Affairs Council. Council of the European Union",
+    "ECU": "European Currency Unit ",
+    "EDF": "expected default frequency",
+    "EDI": "electronic data interchange ",
+    "EDIS": "European Deposit Insurance Scheme",
+    "EDP": "excessive deficit procedure ",
+    "EDW": "European Data Warehouse",
+    "EEA": "European Economic Area ",
+    "EER": "effective exchange rate ",
+    "EFC": "Economic and Financial Committee ",
+    "EFSF": "European Financial Stability Facility ",
+    "EFSM": "European Financial Stabilisation Mechanism ",
+    "EIOPA": "European Insurance and Occupational Pensions Authority",
+    "EL": "Expected Loss",
+    "ELB": "effective lower bound",
+    "ELBE": "Expected Loss Best Estimate",
+    "ELMI": "electronic money institution",
+    "EMIR": "European Market Infrastructure Regulation",
+    "EMMS": "Euro Money Market Survey",
+    "EMS": "European Monetary System ",
+    "EMU": "Economic and Monetary Union ",
+    "EONIA": "euro overnight index average",
+    "ERM II": "exchange rate mechanism II",
+    "ERPB": "Euro Retail Payments Board",
+    "ESA": "European Supervisory Authority",
+    "ESA 2010": "European System of Accounts 2010 ",
+    "ESA 95": "European System of Accounts 1995 ",
+    "ESCB": "European System of Central Banks",
+    "ESCG": "European Systemic Cyber Group",
+    "ESFS": "European System of Financial Supervision",
+    "ESM": "European Stability Mechanism",
+    "ESMA": "European Securities and Markets Authority",
+    "ESRB": "European Systemic Risk Board",
+    "ETF": "exchange-traded fund",
+    "EUCLID": "European centralised infrastructure for supervisory data",
+    "EURIBOR": "euro interbank offered rate",
+    "€STR": "euro short-term rate",
+    "EVE": "economic value of equity",
+    "f.o.b.": "Free on board at the exporter’s border",
+    "FINREP": "financial reporting",
+    "FMI": "financial market infrastructure",
+    "FOLTF": "failing or likely to fail",
+    "FOMC": "Federal Open Market Committee",
+    "FRA": "forward rate agreement",
+    "FSB": "Financial Stability Board",
+    "FSR": "Financial Stability Review",
+    "FTS": "funds transfer system",
+    "FVA": "fair value accounting",
+    "FVC": "financial vehicle corporation",
+    "FX": "foreign exchange",
+    "G-SIB": "global systemically important bank",
+    "G-SII": "global systemically important institution",
+    "GAAP": "generally accepted accounting principles",
+    "GDP": "gross domestic product",
+    "HICP": "Harmonised Index of Consumer Prices",
+    "HLEG": "High-Level Expert Group on Sustainable Finance",
+    "HoM": "Head of Mission",
+    "HQLA": "high-quality liquid asset",
+    "i.i.p.": "international investment position",
+    "IAIG": "internationally active insurance group",
+    "IAIS": "International Association of Insurance Supervisors",
+    "IAS": "International Accounting Standards",
+    "IBAN": "International Bank Account Number",
+    "IC": "internal capital",
+    "ICAAP": "Internal Capital Adequacy Assessment Process",
+    "ICMA": "International Capital Market Association",
+    "ICPFs": "insurance corporations and pension funds",
+    "ICR": "interest coverage ratio",
+    "ICS": "Insurance Capital Standard",
+    "ICSD": "international central securities depository",
+    "IF": "investment fund",
+    "IFRS": "International Financial Reporting Standards",
+    "IFTS": "interbank funds transfer system",
+    "ILAAP": "Internal Liquidity Adequacy Assessment Process",
+    "ILO": "International Labour Organization",
+    "ILS": "inflation-linked swap",
+    "IMAS": "SSM Information Management System",
+    "IMF": "International Monetary Fund",
+    "IMI": "internal model investigation",
+    "IOSCO": "International Organization of Securities Commissions",
+    "IPS": "institutional protection scheme",
+    "IRB": "internal ratings-based",
+    "IRBA": "internal ratings-based approach",
+    "IRR": "internal rate of return",
+    "IRRBB": "interest rate risk in the banking book",
+    "IRT": "Internal Resolution Team",
+    "ISIN": "International Securities Identification Number",
+    "ITS": "Implementing Technical Standards",
+    "JSS": "Joint Supervisory Standards",
+    "JST": "Joint Supervisory Team",
+    "JSTC": "Joint Supervisory Team coordinator",
+    "KRI": "key risk indicator",
+    "LCBG": "large and complex banking group",
+    "LCR": "liquidity coverage ratio",
+    "LGD": "loss-given-default",
+    "LSI": "less significant institution",
+    "LSTI": "loan service-to-income",
+    "LTD": "loan-to-deposit",
+    "LTG": "long-term guarantee",
+    "LTI": "loan-to-income",
+    "LTRO": "longer-term refinancing operation",
+    "LTSF": "loan-to-stable-funding",
+    "LTV": "loan-to-value",
+    "M&A": "mergers and acquisitions",
+    "MDA": "maximum distributable amount",
+    "MFI": "monetary financial institution",
+    "MiFID": "Markets in Financial Instruments Directive",
+    "MiFIR": "Markets in Financial Instruments Regulation",
+    "MIP": "macroeconomic imbalance procedure",
+    "MMF": "money market fund",
+    "MMS": "money market statistics",
+    "MMSR": "money market statistical reporting",
+    "MPC": "Monetary Policy Committee",
+    "MREL": "minimum requirement for own funds and eligible liabilities",
+    "MSC": "merchant service charge",
+    "NAV": "net asset value",
+    "NBNI": "non-bank, non-insurance",
+    "NCA": "national competent authority",
+    "NCB": "national central bank",
+    "NDA": "national designated authority",
+    "NFC": "non-financial corporation",
+    "NFCI": "net fee and commission income",
+    "NII": "net interest income",
+    "NIRP": "negative interest rate policy",
+    "NPE": "non-performing exposure",
+    "NPLs": "non-performing loans",
+    "NRA": "national resolution authority",
+    "NSA": "national supervisory authority",
+    "NSFR": "net stable funding ratio",
+    "O&D": "options and discretions",
+    "O-SII": "other systemically important institution",
+    "OECD": "Organisation for Economic Co-operation and Development",
+    "OFI": "other financial institution",
+    "OIS": "overnight index swap",
+    "OJ": "Official Journal of the European Union",
+    "ORC": "overall recovery capacity",
+    "OSI": "on-site inspection",
+    "OTC": "over-the-counter",
+    "P&L": "profit and loss",
+    "P/E": "price/earnings (ratio)",
+    "P2G": "Pillar 2 guidance",
+    "P2P payment": "peer-to-peer payment",
+    "P2R": "Pillar 2 requirement",
+    "PCE": "personal consumption expenditure",
+    "PD": "probability of default",
+    "PE-ACH": "pan-European automated clearing house",
+    "PIN": "personal identification number",
+    "PPI": "prudential policy index",
+    "PPP": "purchasing power parity",
+    "PQD": "public quantitative disclosure",
+    "PSPP": "public sector purchase programme",
+    "PvP": "payment versus payment",
+    "QE": "quantitative easing",
+    "RAROC": "risk-adjusted return on capital",
+    "RAS": "risk appetite statement",
+    "repo": "repurchase agreement, repurchase operation",
+    "ROA": "return on assets",
+    "ROE": "return on equity",
+    "RORAC": "return on risk-adjusted capital",
+    "RRE": "residential real estate",
+    "RTGS system": "real-time gross settlement system",
+    "RTS": "Regulatory Technical Standards",
+    "RWA": "risk-weighted asset",
+    "S&P": "Standard & Poor’s",
+    "SBBS": "sovereign bond-backed security",
+    "SCR": "Solvency Capital Requirement",
+    "SDR": "special drawing right",
+    "SEP": "Supervisory Examination Programme",
+    "SEPA": "Single Euro Payments Area",
+    "SFT": "securities financing transaction",
+    "SGP": "Stability and Growth Pact",
+    "SI": "significant institution",
+    "SII": "systemically important institution",
+    "SIPS": "systemically important payment system",
+    "SMEs": "small and medium-sized enterprises",
+    "SPV": "special-purpose vehicle",
+    "SQA": "Supervisory Quality Assurance",
+    "SRB": "systemic risk buffer",
+    "SREP": "Supervisory Review and Evaluation Process",
+    "SRF": "Single Resolution Fund",
+    "SRM": "Single Resolution Mechanism",
+    "SRMR": "Single Resolution Mechanism Regulation",
+    "SSG": "SSM Simplification Group",
+    "SSM": "Single Supervisory Mechanism",
+    "SSMR": "Single Supervisory Mechanism Regulation",
+    "SSS": "securities settlement system",
+    "STE": "Short Term Exercise",
+    "STP": "straight-through processing",
+    "T2": "Tier 2",
+    "T2S": "TARGET2-Securities",
+    "TFEU": "Treaty on the Functioning of the European Union",
+    "TIPS": "TARGET instant payment settlement",
+    "TLAC": "total loss-absorbing capacity",
+    "TLTRO": "targeted longer-term refinancing operation",
+    "TREA": "total risk exposure amount",
+    "TRIM": "targeted review of internal models",
+    "TRN": "transaction reference number",
+    "TSCG": "Treaty on Stability, Coordination and Governance in the Economic and Monetary Union",
+    "UL": "Unexpeccted Loss",
+    "TSCR": "total SREP capital requirement (P1R+P2R)",
+    "UCITS": "undertaking for collective investment in transferable securities",
+    "ULCM": "Unit labour costs in the manufacturing sector.",
+    "ULCT": "Unit labour costs in the total economy.",
+    "VaR": "value at risk",
+    "VIX": "Chicago Board Options Exchange’s Volatility Index",
+    "XML": "Extensible Markup Language"
+}

processing.py

@@ -0,0 +1,218 @@
+from abc import ABC, abstractmethod
+from pathlib import Path
+from urllib.parse import urlparse
+import requests
+import fitz
+import io
+import re
+import hashlib
+import os
+
+
+class FileTypeError(Exception):
+    """Raised when the file type does not match the expected file type."""
+
+
+class FileSchemeError(Exception):
+    """Raised when the file scheme does not match the expected file scheme."""
+
+
+class FileProcessor(ABC):
+    type = None
+
+    def __init__(self, path):
+        self.path = path
+        self.file_scheme = self._get_file_scheme()
+        self.__class__._check_file_type(path)
+
+    @abstractmethod
+    def get_file_data(self):
+        pass
+
+    @abstractmethod
+    def _get_file_metadata(self):
+        pass
+
+    @abstractmethod
+    def _get_file_paragraphs(self):
+        pass
+
+    @classmethod
+    def _check_file_type(cls, path):
+        file_type = Path(path).suffix.lower()[1:]
+        if file_type != cls.type:
+            raise FileTypeError(
+                f"Invalid file type. {cls.__name__} expects a {cls.type} file"
+            )
+
+    def _get_file_scheme(self):
+        parsed_path = urlparse(self.path)
+        if (
+            not parsed_path.scheme
+            or parsed_path.scheme.lower() == "file"
+            or os.path.isfile(self.path)
+        ):
+            return "local"
+        elif parsed_path.scheme.lower() in ["http", "https", "ftp"]:
+            return "url"
+        else:
+            raise FileSchemeError("Unknown scheme")
+
+    def _preprocess_text(self, text):
+        text = text.replace("\n", " ")
+        text = re.sub("\s+", " ", text)
+        text = text.encode("utf-8", "ignore").decode("utf-8", "ignore")
+        return text
+
+    def _generate_hash(self, string):
+        hash_object = hashlib.md5()
+        hash_object.update(string.encode("utf-8", "ignore"))
+        hex_dig = hash_object.hexdigest()
+
+        return hex_dig
+
+    def generate_paragraphs():
+        raise NotImplementedError
+
+    def generate_metadata():
+        raise NotImplementedError
+
+
+class PDFProcessor(FileProcessor):
+    type = "pdf"
+
+    def __init__(self, path):
+        super().__init__(path)
+
+    def get_file_data(self, merge_length=200):
+        file = self._open_file()
+
+        file_metadata = self._get_file_metadata(file)
+        file_paragraphs = self._get_file_paragraphs(
+            file, file_metadata, start_page=1, end_page=None, merge_length=merge_length
+        )
+
+        file.close()
+
+        return file_metadata, file_paragraphs
+
+    def _get_file_metadata(self, file):
+        file_metadata = dict()
+
+        metadata = file.metadata
+
+        unique_string = str(Path(self.path).name) + metadata["title"]
+
+        file_metadata["id"] = self._generate_hash(unique_string)
+        file_metadata["title"] = metadata["title"]
+        file_metadata["author"] = metadata["author"]
+        file_metadata["subject"] = metadata["subject"]
+        file_metadata["creation_date"] = metadata["creationDate"]
+        file_metadata["modification_date"] = metadata["modDate"]
+        file_metadata["n_pages"] = file.page_count
+        if self.file_scheme == "local":
+            file_metadata["url"] = str(Path(self.path).resolve())
+        else:
+            file_metadata["url"] = self.path
+        file_metadata["file_name"] = Path(self.path).name
+        file_metadata["short_name"] = Path(self.path).name
+        file_metadata["release_date"] = ""
+        file_metadata["report_type"] = ""
+        file_metadata["source"] = ""
+
+        return file_metadata
+
+    def _get_file_paragraphs(
+        self, file, file_metadata, start_page=1, end_page=None, merge_length=200
+    ):
+        if end_page is None:
+            end_page = file_metadata["n_pages"]
+
+        file_paragraphs = []
+
+        for page_num in range(start_page - 1, end_page):
+            page = file.load_page(page_num)
+            blocks = page.get_text("blocks")
+
+            for block in blocks:
+                paragraph = self._process_block(
+                    block, page, page_num + start_page, file_metadata["id"]
+                )
+                if paragraph is None:
+                    continue
+
+                first_char = paragraph["content"][0]
+                if len(file_paragraphs) > 0:
+                    if (
+                        len(file_paragraphs[-1]["content"]) + len(paragraph["content"])
+                        < merge_length
+                    ) or (first_char.islower() and first_char.isalpha()):
+                        file_paragraphs[-1]["content"] += " " + paragraph["content"]
+                        file_paragraphs[-1]["length"] = len(
+                            file_paragraphs[-1]["content"]
+                        )
+                    else:
+                        file_paragraphs.append(paragraph)
+                else:
+                    file_paragraphs.append(paragraph)
+
+        return file_paragraphs
+
+    def _open_file(self):
+        if self.file_scheme == "url":
+            response = requests.get(self.path)
+            file = fitz.open(stream=io.BytesIO(response.content), filetype="pdf")
+        elif self.file_scheme == "local":
+            file = fitz.open(self.path)
+        return file
+
+    def _process_block(self, block, page, page_number, file_id):
+        x0, y0, x1, y1, content, block_no, block_type = block
+
+        if content.isspace() or block_type == 1:
+            return None
+
+        content = self._preprocess_text(content)
+        unique_content_string = "_".join(map(str, block))
+        paragraph_id = self._generate_hash(unique_content_string)
+
+        w, h = page.rect.width, page.rect.height
+        paragraph = {
+            "id": paragraph_id,
+            "document_id": file_id,
+            "content_type": "text" if block_type == 0 else "image",
+            "content": content,
+            "length": len(content),
+            "idx_block": block_no,
+            "page_number": page_number,
+            "x0": x0 / h,
+            "y0": y0 / w,
+            "x1": x1 / h,
+            "y1": y1 / w,
+        }
+
+        return paragraph
+
+
+class HTMLProcessor(FileProcessor):
+    type = "html"
+
+    def __init__(self, path):
+        super().__init__(path)
+
+    def get_file_data(self):
+        pass
+
+    def _get_file_metadata(self):
+        pass
+
+    def _get_file_paragraphs(self):
+        pass
+
+    def _open_file(self):
+        if self.file_scheme == "url":
+            response = requests.get(self.path)
+            file = response.text
+        elif self.file_scheme == "local":
+            file = open(self.path, "r").read()
+        return file

requirements.txt

@@ -0,0 +1,12 @@
+altair==4.2.2
+datasets==2.12.0
+faiss-cpu==1.7.4
+gradio==3.39.0
+gradio_client==0.3.0
+openai==0.27.0
+PyMuPDF==1.22.3
+python-dotenv==1.0.0
+sentence-transformers==2.2.2
+torch==2.0.1
+matplotlib==3.7.1
+tiktoken==0.4.0

text_embedder.py

@@ -0,0 +1,82 @@
+from abc import ABC, abstractmethod
+
+import pandas as pd
+import torch
+from datasets import load_from_disk
+from sentence_transformers import SentenceTransformer
+
+# from finbert_embedding.embedding import FinbertEmbedding
+
+
+class TextEmbedder(ABC):
+    def __init__(self, model_name, paragraphs_path, device, load_existing_index=False):
+        self.dataset = load_from_disk(paragraphs_path)
+        self.model = self._load_model(model_name, device)
+
+        assert len(self.dataset) > 0, "The loaded dataset is empty !!"
+
+        if load_existing_index == True:
+            self.dataset.load_faiss_index(
+                "embeddings", f"{paragraphs_path}/index.faiss"
+            )
+
+    def generate_paragraphs_embedding(self):
+        self.dataset = self.dataset.map(
+            lambda x: {"embeddings": self._generate_embeddings(x["content"])}
+        )
+
+    def save_embeddings(self, output_path):
+        self.dataset.add_faiss_index(column="embeddings")
+        self.dataset.save_faiss_index("embeddings", f"{output_path}/index.faiss")
+
+    def retrieve_faiss(self, query: str, k_total: int, threshold: int):
+        question_embedding = self._generate_embeddings(query)
+        scores, samples = self.dataset.get_nearest_examples(
+            "embeddings", question_embedding, k=k_total
+        )
+        passages_df = pd.DataFrame(samples)
+        passages_df["scores"] = scores / 100
+        passages_df = passages_df[passages_df["scores"] > threshold]
+        passages_df = passages_df.sort_values(by=["scores"], ascending=False)
+
+        if len(passages_df) == 0:
+            return [], []
+
+        contents = passages_df["content"].tolist()
+        meta = passages_df.drop(columns=["content"]).to_dict(orient="records")
+        passages = []
+        for i in range(len(contents)):
+            passages.append({"content": contents[i], "meta": meta[i]})
+        return passages, passages_df["scores"].values
+
+    def retrieve_elastic(self, query: str, k_total: int, threshold: int):
+        raise NotImplementedError
+
+    @abstractmethod
+    def _load_model(self, model_name: str, device: str):
+        pass
+
+    @abstractmethod
+    def _generate_embeddings(self, text: str):
+        pass
+
+
+class SentenceTransformersTextEmbedder(TextEmbedder):
+    def _load_model(self, model_name: str, device: str):
+        model = SentenceTransformer(model_name)
+        torch_device = torch.device(device)
+        model.to(torch_device)
+        return model
+
+    def _generate_embeddings(self, text: str):
+        return self.model.encode(text)
+
+
+# class FinBertTextEmbedder(TextEmbedder):
+#     def _load_model(self, model_name: str, device: str):
+#         model = FinbertEmbedding(device=device)
+#         return model
+
+#     def _generate_embeddings(self, text: str):
+#         output = self.model.sentence_vector(text)
+#         return output.cpu().numpy()

utils.py

@@ -0,0 +1,217 @@
+import json
+import openai
+import re
+from config import CFG_APP
+from text_embedder import SentenceTransformersTextEmbedder
+from datetime import datetime
+import tiktoken
+
+doc_metadata = json.load(open(CFG_APP.DOC_METADATA_PATH, "r"))
+# Embedding Model
+if "sentence-transformers" in CFG_APP.EMBEDDING_MODEL:
+    text_embedder = SentenceTransformersTextEmbedder(
+        model_name=CFG_APP.EMBEDDING_MODEL,
+        paragraphs_path=CFG_APP.DATA_FOLDER,
+        device=CFG_APP.DEVICE,
+        load_existing_index=True,
+    )
+else:
+    raise ValueError("Embedding model not found !")
+
+
+# Util Functions
+def retrieve_doc_metadata(doc_metadata, doc_id):
+    for meta in doc_metadata:
+        if meta["id"] == doc_id:
+            return meta
+
+
+def get_reformulation_prompt(query: str) -> list:
+    return [
+        {
+            "role": "user",
+            "content": f"""{CFG_APP.REFORMULATION_PROMPT}
+            ---
+            query: {query}
+            standalone question: """,
+        }
+    ]
+
+
+def make_pairs(lst):
+    """From a list of even lenght, make tupple pairs
+    Args:
+        lst (list): a list of even lenght
+    Returns:
+        list: the list as tupple pairs
+    """
+    assert not (l := len(lst) % 2), f"your list is of lenght {l} which is not even"
+    return [(lst[i], lst[i + 1]) for i in range(0, len(lst), 2)]
+
+
+def make_html_source(paragraph, meta_doc, i):
+    content = paragraph["content"]
+    meta_paragraph = paragraph["meta"]
+    return f"""
+<div class="card" id="document-{i}">
+    <div class="card-content">
+        <h2>Doc {i} - {meta_doc['short_name']} - Page {meta_paragraph['page_number']}</h2>
+        <p>{content}</p>
+    </div>
+    <div class="card-footer">
+        <span>{meta_doc['short_name']}</span>
+        <a href="{meta_doc['url']}#page={meta_paragraph['page_number']}" target="_blank" class="pdf-link">
+            <span role="img" aria-label="Open PDF">🔗</span>
+        </a>
+    </div>
+</div>
+"""
+
+
+def preprocess_message(text: str) -> str:
+    return re.sub(
+        r"\[doc (\d+)\]",
+        lambda match: f'<a href="#do-{match.group(1)}">{match.group(0)}</a>',
+        text,
+    )
+
+
+def parse_glossary(query):
+    file = "glossary.json"
+    glossary = json.load(open(file, "r"))
+    words_query = query.split(" ")
+    for i, word in enumerate(words_query):
+        for key in glossary.keys():
+            if word.lower() == key.lower():
+                words_query[i] = words_query[i] + f" ({glossary[key]})"
+    return " ".join(words_query)
+
+
+def num_tokens_from_string(string: str, encoding_name: str) -> int:
+    encoding = tiktoken.encoding_for_model(encoding_name)
+    num_tokens = len(encoding.encode(string))
+    return num_tokens
+
+
+def chat(
+    query: str,
+    history: list,
+    threshold: float = CFG_APP.THRESHOLD,
+    k_total: int = CFG_APP.K_TOTAL,
+) -> tuple:
+    """retrieve relevant documents in the document store then query gpt-turbo
+    Args:
+        query (str): user message.
+        history (list, optional): history of the conversation. Defaults to [system_template].
+        report_type (str, optional): should be "All available" or "IPCC only". Defaults to "All available".
+        threshold (float, optional): similarity threshold, don't increase more than 0.568. Defaults to 0.56.
+    Yields:
+        tuple: chat gradio format, chat openai format, sources used.
+    """
+
+    reformulated_query = openai.ChatCompletion.create(
+        model=CFG_APP.MODEL_NAME,
+        messages=get_reformulation_prompt(parse_glossary(query)),
+        temperature=0,
+        max_tokens=CFG_APP.MAX_TOKENS_REF_QUESTION,
+    )
+
+    reformulated_query = reformulated_query["choices"][0]["message"]["content"]
+    if len(reformulated_query.split("\n")) == 2:
+        reformulated_query, language = reformulated_query.split("\n")
+        language = language.split(":")[1].strip()
+    else:
+        reformulated_query = reformulated_query.split("\n")[0]
+        language = "English"
+
+    sources, scores = text_embedder.retrieve_faiss(
+        reformulated_query,
+        k_total=k_total,
+        threshold=threshold,
+    )
+    if CFG_APP.DEBUG == True:
+        print("Scores : \n", scores)
+
+    messages = history + [{"role": "user", "content": query}]
+
+    if len(sources) > 0:
+        docs_string = []
+        docs_html = []
+
+        num_tokens = num_tokens_from_string(CFG_APP.SOURCES_PROMPT, CFG_APP.MODEL_NAME)
+
+        for i, data in enumerate(sources, 1):
+            meta_doc = retrieve_doc_metadata(doc_metadata, data["meta"]["document_id"])
+            doc_content = f"📃 Doc {i}: \n{data['content']}"
+            num_tokens_doc = num_tokens_from_string(doc_content, CFG_APP.MODEL_NAME)
+            if num_tokens + num_tokens_doc > CFG_APP.MAX_TOKENS_API:
+                break
+            num_tokens += num_tokens_doc
+            docs_string.append(doc_content)
+            docs_html.append(make_html_source(data, meta_doc, i))
+
+        docs_string = "\n\n".join(
+            [f"Query used for retrieval:\n{reformulated_query}"] + docs_string
+        )
+        docs_html = "\n\n".join(
+            [f"Query used for retrieval:\n{reformulated_query}"] + docs_html
+        )
+        messages.append(
+            {
+                "role": "system",
+                "content": f"{CFG_APP.SOURCES_PROMPT}\n\n{docs_string}\n\nAnswer in {language}:",
+            }
+        )
+
+        if CFG_APP.DEBUG == True:
+            print(f" 👨‍💻 question asked by the user : {query}")
+            print(f" 🕛 time : {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}")
+
+            print(" 🔌 messages sent to the API :")
+            api_messages = [
+                {"role": "system", "content": CFG_APP.INIT_PROMPT},
+                {"role": "user", "content": reformulated_query},
+                {
+                    "role": "system",
+                    "content": f"{CFG_APP.SOURCES_PROMPT}\n\n{docs_string}\n\nAnswer in {language}:",
+                },
+            ]
+            for message in api_messages:
+                print(
+                    f"length : {len(message['content'])}, content : {message['content']}"
+                )
+
+        response = openai.ChatCompletion.create(
+            model=CFG_APP.MODEL_NAME,
+            messages=[
+                {"role": "system", "content": CFG_APP.INIT_PROMPT},
+                {"role": "user", "content": reformulated_query},
+                {
+                    "role": "system",
+                    "content": f"{CFG_APP.SOURCES_PROMPT}\n\n{docs_string}\n\nAnswer in {language}:",
+                },
+            ],
+            temperature=0,  # deterministic
+            stream=True,
+            max_tokens=CFG_APP.MAX_TOKENS_ANSWER,
+        )
+        complete_response = ""
+        messages.pop()
+
+        messages.append({"role": "assistant", "content": complete_response})
+
+        for chunk in response:
+            chunk_message = chunk["choices"][0]["delta"].get("content")
+            if chunk_message:
+                complete_response += chunk_message
+                complete_response = preprocess_message(complete_response)
+                messages[-1]["content"] = complete_response
+                gradio_format = make_pairs([a["content"] for a in messages[1:]])
+                yield gradio_format, messages, docs_html
+
+    else:
+        docs_string = "⚠️ No relevant passages found in this report"
+        complete_response = "**⚠️ No relevant passages found in this report, you may want to ask a more specific question.**"
+        messages.append({"role": "assistant", "content": complete_response})
+        gradio_format = make_pairs([a["content"] for a in messages[1:]])
+        yield gradio_format, messages, docs_string