Add demo

.gitattributes

@@ -7,6 +7,7 @@
 *.gz filter=lfs diff=lfs merge=lfs -text
 *.h5 filter=lfs diff=lfs merge=lfs -text
 *.joblib filter=lfs diff=lfs merge=lfs -text
+*.jpeg filter=lfs diff=lfs merge=lfs -text
 *.lfs.* filter=lfs diff=lfs merge=lfs -text
 *.mlmodel filter=lfs diff=lfs merge=lfs -text
 *.model filter=lfs diff=lfs merge=lfs -text
@@ -19,6 +20,7 @@
 *.pb filter=lfs diff=lfs merge=lfs -text
 *.pickle filter=lfs diff=lfs merge=lfs -text
 *.pkl filter=lfs diff=lfs merge=lfs -text
+*.png filter=lfs diff=lfs merge=lfs -text
 *.pt filter=lfs diff=lfs merge=lfs -text
 *.pth filter=lfs diff=lfs merge=lfs -text
 *.rar filter=lfs diff=lfs merge=lfs -text

README.md

@@ -5,7 +5,7 @@ colorFrom: green
 colorTo: pink
 sdk: gradio
 sdk_version: 3.32.0
-app_file: app.py
+app_file: gradio_demo.py
 pinned: false
 ---
 

gradio_demo.py

@@ -0,0 +1,127 @@
+import gradio as gr
+from langchain.chat_models import ChatOpenAI
+from langchain.schema import HumanMessage
+from utils import SyncStreamingLLMCallbackHandler
+from anyio.from_thread import start_blocking_portal
+from queue import Queue
+import prompt, tiktoken
+
+def num_token(string: str) -> int:
+    """Returns the number of tokens in a text string."""
+    encoding = tiktoken.get_encoding('cl100k_base')
+    num_tokens = len(encoding.encode(string))
+    return num_tokens
+
+def send_message(history, temp, top_p, fp, pp):
+    q = Queue()
+    job_done = object()
+    def task():
+        llm = ChatOpenAI(
+            model_name="gpt-3.5-turbo",
+            streaming=True,
+            temperature=temp,
+            model_kwargs = {
+                'top_p':top_p,
+                'frequency_penalty':fp,
+                'presence_penalty':pp
+            }
+        )
+        query_llm = ChatOpenAI(
+            model_name="gpt-3.5-turbo",
+            temperature = 0
+        )
+        query_template = prompt.search_query_gradio(history)
+        search_query = query_llm([HumanMessage(content=query_template)]).content
+        print('query: ', search_query)
+        answer_template = prompt.question_answer_gradio(history, search_query)
+        print('final temp: \n\n', answer_template, f'\nprompt total: {num_token(answer_template)}')
+        llm([HumanMessage(content=answer_template)], callbacks=[SyncStreamingLLMCallbackHandler(q)])
+        q.put(job_done)
+        return
+
+    with start_blocking_portal() as portal:
+        portal.start_task_soon(task)
+
+        content = ""
+        while True:
+            next_token = q.get(True, timeout=10)
+            if next_token is job_done:
+                return history
+
+            content += next_token
+            latest = history[-1]
+            latest[1] = content
+            yield history
+
+
+def append_user_message(message, history):
+    "Add user message to chain/history and clear input field"
+    print(history)
+    history.append([message, None])
+    return "", history
+
+css = """
+.contain { display: flex; flex-direction: column; height: 100%;}
+#chatbot { flex-grow: 1;}
+#chatbot .user {text-align: right;}
+
+.contain { display: flex; flex-direction: column; height: 100%;}
+#query_ref { flex-grow: 1; }
+"""
+
+
+with gr.Blocks(css=css, theme=gr.themes.Default(primary_hue="orange")) as demo:
+
+    with gr.Row(elem_id="whole_page"):
+        with gr.Column(elem_id="sidebar", scale=0.2, min_width=0):
+            # TODO temperature?
+            gr.Markdown('## City of Lake Elsinore Proposal Demo', show_label=False)
+            with gr.Row(elem_id="logo_section1"):
+                with gr.Column(elem_id="logo_col1", scale=0.5, min_width=0):
+                    gr.HTML("""<div style="display: flex; justify-content: center; align-items: center"><img src="./file=icon_1-modified.png" alt="ThinkCol" width="75" height="87" /></a></div>""")
+                with gr.Column(elem_id="logo_col2", scale=0.5, min_width=0):
+                    gr.HTML("""<div style="display: flex; justify-content: center; align-items: center"><img src="./file=icon_2-modified.png" alt="ThinkCol" width="75" height="87" /></a></div>""")
+            gr.Markdown('### Chatbot Settings', show_label=False)
+            temp = gr.Slider(maximum=1, label='Temperature')
+            top_p = gr.Slider(maximum=1, label='Top_p', value=1, interactive=True, visible='primary')
+            fp = gr.Slider(maximum=2, label='Frequency Penalty')
+            pp = gr.Slider(maximum=2, label='Presence Penalty')
+
+        with gr.Column(elem_id="main_chat_interface", scale=0.5):
+            chatbot = gr.Chatbot([], elem_id="chatbot")
+            with gr.Row(elem_id="box_split"):
+                with gr.Column(elem_id="enter_box",scale=0.85):
+                    txt = gr.Textbox(
+                        elem_id='input',
+                        show_label=False,
+                        placeholder="Enter text and press enter, or upload an image"
+                    ).style(container=False)
+                    txt.submit(append_user_message, [txt, chatbot], [txt, chatbot]) \
+                        .then(send_message, [chatbot, temp, top_p, fp, pp], [chatbot])
+
+                with gr.Column(elem_id="send_box",scale=0.15, min_width=0):
+                    btn = gr.Button('Send', elem_id='send', variant='primary')
+                    btn.click(append_user_message, [txt, chatbot], [txt, chatbot]) \
+                        .then(send_message, [chatbot, temp, top_p, fp, pp], [chatbot])
+
+
+        # with gr.Column(elem_id="main_chat_interface", scale=0.3):
+        #     with gr.Tab("Search Query"):
+        #         query_ref = gr.TextArea(
+        #                 value='',
+        #                 interactive=False,
+        #                 elem_id='query_ref',
+        #                 show_label=False,
+        #             ).style(container=False)
+        #     with gr.Tab("Prompt"):
+        #         prompt_ref = gr.TextArea(
+        #                 value='',
+        #                 interactive=False,
+        #                 elem_id='prompt_ref',
+        #                 show_label=False,
+        #             ).style(container=False)
+        #     print(query_ref, prompt_ref)
+
+demo.queue()
+if __name__ == "__main__":
+    demo.launch()

prompt.py

@@ -0,0 +1,202 @@
+import os
+import tiktoken
+import requests, json
+
+
+def num_token(string: str) -> int:
+    """Returns the number of tokens in a text string."""
+    encoding = tiktoken.get_encoding('cl100k_base')
+    num_tokens = len(encoding.encode(string))
+    return num_tokens
+
+def qa_prompt(context_str, query_str):
+    temp = (
+        "Context information is below. \n"
+        "---------------------\n"
+        "{context_str}"
+        "\n---------------------\n"
+        "Given the context information and not prior knowledge, "
+        "answer the question: {query_str}\n"
+    )
+    temp = temp.format(context_str=context_str, query_str=query_str)
+    print('qa_prompt token: ', num_token(temp))
+    return temp
+
+
+
+def refine_prompt(context_str, query_str, existing_answer):
+    temp = (
+        "The original question is as follows: {query_str}\n"
+        "We have provided an existing answer: {existing_answer}\n"
+        "Given the new context, refine the original answer to better "
+        "answer the question. However, "
+        "------------\n"
+        "{context_str}\n"
+        "------------\n"
+        "If the context isn't useful, return the original answer as new answer."
+        "New answer:"
+    )
+    temp = temp.format(
+        context_str=context_str,
+        query_str=query_str,
+        existing_answer=existing_answer
+    )
+    print('qa_prompt token: ', num_token(temp))
+    return temp
+
+
+def qa_prompt_t(query_str):
+    temp = (
+        "Context information is below. \n"
+        "---------------------\n"
+        "{text}"
+        "\n---------------------\n"
+        "Given the context information and not prior knowledge, "
+        "answer the question: {query_str}\n"
+    )
+    return temp.replace('{query_str}', query_str)
+
+
+def refine_prompt_t(query_str):
+    temp = (
+        "The original question is as follows: {query_str}\n"
+        "We have provided an existing answer: {existing_answer}\n"
+        "Given the new context, refine the original answer to better "
+        "answer the question. However, "
+        "------------\n"
+        "{text}\n"
+        "------------\n"
+        "If the context isn't useful, return the original answer as new answer."
+        "New answer:"
+    )
+    return temp.replace('{query_str}', query_str)
+
+def language_detection(query_str):
+    detect_template = (
+        "Determine if the QUERY want to be answered in English/繁體中文/简体中文 (ONLY choose one).\n"
+        f"QUERY: {query_str}"
+        "LANGUAGE of the answer should be (default English if you don't know):"
+    )
+    return detect_template
+
+# rephrased text is the current text.
+def rephrase_query(query_str, history, company, language):
+    rephrase_template = (
+        f"Given the following conversation and a new input from user, rephrase it (in {language}) to a standalone input.\n\n"
+
+        "Chat History:\n"
+        f"{history}\n"
+
+        f"User (New Input): {query_str}\n"
+        "Standalone Input (rephrase according to User's perspective):"
+    )
+    return rephrase_template
+
+
+def rephrase_query_hkas(query_str, history, language):
+    rephrase_template = (
+        f"Rewrite the new QUERY in {language} given the context of the previous QA history, "
+        "such that the objective and subject should be clear in the REWRITED QUERY.\n\n "
+
+        "QA history:\n"
+        "===\n"
+        f"{history}\n"
+        "===\n\n"
+
+        f"QUERY: {query_str} \n"
+        f"REWRITED QUERY ONLY (no description):"
+    )
+    return rephrase_template
+
+def rephrase_query_chat(query, history, language):
+    rephrase_template = (
+        f"Given the following conversation and a new input from user, rephrase it (in {language}) to a new standalone input.\n\n"
+
+        "Chat History:\n"
+        f"{history}\n"
+
+        f"New Input: {query}\n"
+        "Standalone Input (New Input perspective):"
+    )
+    return rephrase_template
+
+
+def search_query_gradio(chat_history):
+    history = []
+    for i in chat_history:
+        if i[1]:
+            history.append(f'User: {i[0]}')
+            history.append(f'Assistant: {i[1]}')
+        else:
+            question = i[0]
+    num = [num_token(record) for record in history]
+    cum_sum = [abs(sum(num[-i-1::]) - 700) for i in range(len(num))]
+    cum_sum = cum_sum[::-1]
+    if cum_sum:
+        idx = cum_sum.index(min(cum_sum))
+        format_history = '\n'.join(i for i in history[idx:])
+    else:
+        format_history = ''
+    print(format_history)
+    template = """
+    Below is a history of the conversation so far, and a new question asked by the user that needs to be answered by searching in a knowledge base. Generate a search query based on the conversation and the new question. Do not include cited source filenames and document names e.g info.txt or doc.pdf or https://web.com in the search query terms. Do not include any text inside [] or <<>> in the search query terms.
+
+    Chat History:
+    {format_history}
+
+    Question:
+    {question}
+
+
+    Search query in the language used in Question:
+    """
+
+    return template.format(format_history=format_history, question=question)
+
+
+def question_answer_gradio(chat_history, search_query):
+
+    docs_string = requests.get(f'http://{os.environ["QUESTION_ANSWER"]}/retrieve_docs/comsec/{search_query}')
+    docs_dict = json.loads(docs_string.json())
+    source_lst = []
+    for i, k in docs_dict.items():
+        source_lst.append(f"{i}: {k}")
+    print('can do source lst: ', source_lst)
+    format_source = '\n\n'.join(source_lst)
+    # TODO number of docs
+
+
+
+
+    history = []
+    for i in chat_history:
+        if i[1]:
+            history.append(f'User: {i[0]}')
+            history.append(f'Assistant: {i[1]}')
+        else:
+            history.append(f'User: {i[0]}')
+    num = [num_token(record) for record in history]
+    cum_sum = [abs(sum(num[-i-1::]) - 700) for i in range(len(num))]
+    cum_sum = cum_sum[::-1]
+    if cum_sum:
+        idx = cum_sum.index(min(cum_sum))
+        format_history = '\n'.join(i for i in history[idx:])
+    else:
+        format_history = ''
+    print('history ok: ', format_history)
+    # Each source has a name followed by colon and the actual information, ALWAYS include the source name for each fact you use in the response. Use square brakets to reference the source, e.g. [info1.txt]. Don't combine sources, list each source separately, e.g. [info1.txt][info2.pdf].
+    template = """<|im_start|>
+    Assistant helps the user with their questions about City of Lake Elsinore Proposal. Be brief in your answers with appropriate tone and emotion. Answer ONLY with the facts listed in the list of sources below, where each source has a name followed by colon and the actual information. If there isn't enough information below, say you don't know. Do not generate answers that don't use the sources below. If asking a clarifying question to the user would help, ask the question. For tabular information return it as an html table. Do not return markdown format.
+
+    Sources:
+    {sources}
+    <|im_end|>
+
+    {chat_history}
+    Assistant:
+    """
+
+    token_for_doc = 3700 - num_token(format_history) - num_token(template)
+    print('token for doc: ', token_for_doc)
+
+    return template.format(sources=format_source, chat_history=format_history)

requirements.txt

@@ -0,0 +1,6 @@
+gradio==3.32.0
+langchain==0.0.183
+tiktoken==0.3.1
+requests==2.28.2
+anyio==3.6.2
+openai

utils.py

@@ -0,0 +1,217 @@
+from langchain.llms import OpenAI
+from langchain.prompts import PromptTemplate
+from langchain.chains.qa_with_sources import load_qa_with_sources_chain
+from langchain.chat_models import ChatOpenAI
+from langchain.schema import HumanMessage
+import tiktoken
+
+
+def num_token(string: str) -> int:
+    """Returns the number of tokens in a text string."""
+    encoding = tiktoken.get_encoding('cl100k_base')
+    num_tokens = len(encoding.encode(string))
+    return num_tokens
+
+
+def retrive_doc_on_token(rephrased_query, db):
+    top_docs = db.similarity_search(rephrased_query, k=10)
+    num = [num_token(doc.page_content) for doc in top_docs]
+    cum_sum = [abs(sum(num[:i+1]) - 2700) for i in range(len(num))]
+    idx = cum_sum.index(min(cum_sum))
+
+    return top_docs[:idx+1]
+
+
+
+
+def multi_docs_qa(query, rephrased_query, db, company, language, temperature):
+    """
+    Return an answer to the query based on multiple documents limited
+    by total token of 3000.
+    """
+    print('temperature: ', temperature)
+    template = """<|im_start|>
+    Manulife's assistant helps the user with their questions about products and services. Be brief in your answers with appropriate tone and emotion. Answer ONLY with the facts listed in the list of sources below. If there isn't enough information below, say you don't know. Do not generate answers that don't use the sources below. If asking a clarifying question to the user would help, ask the question. For tabular information return it as an html table. Do not return markdown format. Each source has a name followed by colon and the actual information, ALWAYS include the source name for each fact you use in the response. Use square brakets to reference the source, e.g. [info1.txt]. Don't combine sources, list each source separately, e.g. [info1.txt][info2.pdf].
+
+    Sources:
+    {sources}
+    <|im_end|>
+
+    {chat_history}
+    """
+
+    docs =  retrive_doc_on_token(query, db)
+    sources = []
+    for i in docs:
+        source_txt = i.metadata['source']
+        source_content = i.page_content
+        add = f"{source_txt}: {source_content}"
+        sources.append(add)
+
+    s_txt = '\n\n'.join(sources)
+    # print('this is docs: ', docs)
+    ch = rephrased_query + f'\nUser: {query}' + '\nAssistant: '
+    final_template = template.format(sources=s_txt, chat_history=ch)
+    print('\n\n', final_template, '\n\n')
+    # PROMPT = PromptTemplate(template=template, input_variables=["summaries", "question"])
+    # llm = OpenAI(temperature=temperature, model_name='gpt-3.5-turbo')
+    # chain = load_qa_with_sources_chain(llm, chain_type="stuff", prompt=PROMPT)
+    # response = chain({"input_documents": docs, "question": query})
+
+    llm = ChatOpenAI(model_name="gpt-3.5-turbo", temperature=temperature)
+    response = llm([HumanMessage(content=final_template)]).content
+
+
+    # return response['output_text'], docs
+    return response, ''
+
+
+
+def multi_docs_qa_hkas(query, rephrased_query, db, language, temperature):
+    """
+    Return an answer to the query based on multiple documents limited
+    by total token of 3000.
+    """
+
+    template = (
+        f"Create a comprehensive and truthful final response in {language}. "
+        "Ask for clearification before answering if the QUESTION is not clear.\n\n"
+
+        "Context (may or may not be useful)"
+        "===\n"
+        "{summaries}\n"
+        "===\n\n"
+
+        "Query:  "
+        "===\n"
+        "{question} "
+        f"({rephrased_query})\n"
+        "===\n\n"
+
+        f"FINAL RESPONSE (in complete sentence):"
+    )
+    docs =  retrive_doc_on_token(query+ f" ({rephrased_query})", db)
+
+
+    PROMPT = PromptTemplate(template=template, input_variables=["summaries", "question"])
+    llm = OpenAI(temperature=temperature, model_name='gpt-3.5-turbo')
+    chain = load_qa_with_sources_chain(llm, chain_type="stuff", prompt=PROMPT)
+    response = chain({"input_documents": docs, "question": query})
+
+    return response['output_text'], docs
+    # return response['output_text'].replace('Manulife', 'Company A').replace('manulife', 'Company A'), docs
+
+
+
+def summary(query, context):
+    template = (
+        "Use the following portion of a long document to see if any of the text is relevant to answer the question. "
+        "Return any relevant text verbatim and 'SOURCE'.\n\n"
+
+        "===\n"
+        "{context}"
+        "===\n\n"
+
+        "Question: {question}\n"
+        "Relevant text, if any:"
+    )
+    return template.format(context=context,question=query)
+
+import asyncio
+
+async def multi_reponse(temperature, messages, docs):
+    chat = ChatOpenAI(temperature=temperature, model_name="gpt-3.5-turbo")
+    responses = await chat.agenerate(messages=messages)
+
+    text = ''
+
+    for i, r in enumerate(responses.generations):
+        if 'N/A' not in r[0].text:
+            text += f"{r[0].text}\nSOURCE: {docs[i].metadata['source']}\n\n"
+
+    print(text)
+
+    return text
+
+
+from typing import Any, Dict, List, Optional, Union
+from types import GeneratorType
+from langchain.callbacks.base import AsyncCallbackHandler, BaseCallbackHandler
+from langchain.schema import AgentAction, AgentFinish, LLMResult
+
+class SyncStreamingLLMCallbackHandler(BaseCallbackHandler):
+    """Callback handler for streaming LLM responses to a queue."""
+
+    def __init__(self, q):
+        self.q = q
+
+    def on_llm_start(
+        self, serialized: Dict[str, Any], prompts: List[str], **kwargs: Any
+    ) -> None:
+        """Do nothing."""
+        pass
+
+    def on_llm_new_token(self, token: str, **kwargs: Any) -> None:
+        self.q.put(token)
+
+    def on_llm_end(self, response: LLMResult, **kwargs: Any) -> None:
+        """Do nothing."""
+        pass
+
+    def on_llm_error(
+        self, error: Union[Exception, KeyboardInterrupt], **kwargs: Any
+    ) -> None:
+        """Do nothing."""
+        pass
+
+    def on_chain_start(
+        self, serialized: Dict[str, Any], inputs: Dict[str, Any], **kwargs: Any
+    ) -> None:
+        """Do nothing."""
+        pass
+
+    def on_chain_end(self, outputs: Dict[str, Any], **kwargs: Any) -> None:
+        """Do nothing."""
+        pass
+
+    def on_chain_error(
+        self, error: Union[Exception, KeyboardInterrupt], **kwargs: Any
+    ) -> None:
+        """Do nothing."""
+        pass
+
+    def on_tool_start(
+        self,
+        serialized: Dict[str, Any],
+        input_str: str,
+        **kwargs: Any,
+    ) -> None:
+        """Do nothing."""
+        pass
+
+    def on_tool_end(
+        self,
+        output: str,
+        color: Optional[str] = None,
+        observation_prefix: Optional[str] = None,
+        llm_prefix: Optional[str] = None,
+        **kwargs: Any,
+    ) -> None:
+        """Do nothing."""
+        pass
+
+    def on_tool_error(
+        self, error: Union[Exception, KeyboardInterrupt], **kwargs: Any
+    ) -> None:
+        """Do nothing."""
+        pass
+
+    def on_agent_action(self, action: AgentAction, **kwargs: Any) -> Any:
+        """Run on agent action."""
+        pass
+
+    def on_agent_finish(
+        self, finish: AgentFinish, color: Optional[str] = None, **kwargs: Any
+    ) -> None:
+        """Run on agent end."""
+        pass