Upload 2 files

arthur_vectorstore.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f48e27fbb868a6f6e54599797183284470462c53a47d5c5efdddcee7378bda8f
+size 6836487

launch_ask_arthur_gradio.py

@@ -0,0 +1,208 @@
+import argparse
+import gradio as gr
+import os
+import pandas as pd
+import pickle
+from typing import List
+
+from langchain.llms import OpenAIChat
+from langchain.prompts import PromptTemplate
+from langchain.chains.question_answering import load_qa_chain
+from langchain.chains import LLMChain, ChatVectorDBChain
+from langchain.callbacks.base import CallbackManager
+from langchain.chains.chat_vector_db.prompts import CONDENSE_QUESTION_PROMPT as SYNTHESIS_PROMPT
+
+
+response_prompt_template = """You are an AI assistant for the machine learning monitoring startup Arthur.  You are 
+given the following extracted parts of a long document and a question. If the question
+includes a request for code, provide a code block directly from the documentation. If you don't know the answer, just
+say "Hmm, I'm not sure." Don't try to make up an answer. If the question is not about Arthur, politely inform them that
+you are tuned to only answer questions about Arthur.
+
+=========
+Example 1:
+
+Question: What data do I need to send to Arthur?
+=========
+**3. What if my data is proprietary? Can I still use Arthur?**
+
+Yes! Arthur offers on-premises installation for customers with data security requirements. By integrating Arthur
+into your business's on-premises stack, you can be confident that all security requirements are met while still
+getting the benefits of the computation and analytics Arthur provides.
+***
+
+**4. What if I don’t have ground truth labels for my data? Or what if I will have the ground truth labels in the future, 
+but they are not available yet?**
+
+You don't need ground truth labels to log your model's inferences with Arthur. 
+
+If your ground truth labels become available after your model's inferences, whether seconds later or years later, 
+Arthur can link these new ground truth values to your model's past predictions, linking the new values by ID to 
+their corresponding inferences already in the Arthur system.
+
+In the meantime, Arthur’s data drift metrics can offer leading indicators of model underperformance to keep you 
+covered if your ground truth labels are delayed or never become available.
+***
+=========
+Answer in Markdown:
+The data you need to get into Arthur is only the inference data - no ground truth is needed, since it can be uploaded
+at a later time. Also, if you have proprietary data, you can install Arthur on-premises to keep your own data security protocols.
+
+=========
+Now the real question:
+Question: {question}
+=========
+{context}
+=========
+Answer in Markdown:"""
+RESPONSE_PROMPT = PromptTemplate(
+    template=response_prompt_template, input_variables=["context", "question"]
+)
+
+
+# load vectorstore of embeddings
+with open("files/vectorstores/arthur_vectorstore.pkl", "rb") as f:
+    global arthur_vectorstore
+    arthur_vectorstore = pickle.load(f)
+
+
+def get_langchain_agent(api_key):
+    os.environ["OPENAI_API_KEY"] = api_key
+
+    manager = CallbackManager([])
+    question_manager = CallbackManager([])
+    stream_manager = CallbackManager([])
+
+    question_gen_llm = OpenAIChat(
+        temperature=0,
+        verbose=True,
+        callback_manager=question_manager,
+    )
+    streaming_llm = OpenAIChat(
+        streaming=True,
+        callback_manager=stream_manager,
+        verbose=True,
+        temperature=0,
+    )
+
+    question_generator = LLMChain(
+        llm=question_gen_llm, prompt=SYNTHESIS_PROMPT, callback_manager=manager
+    )
+    chat_response_generator = load_qa_chain(
+        streaming_llm, chain_type="stuff", prompt=RESPONSE_PROMPT, callback_manager=manager
+    )
+
+    agent = ChatVectorDBChain(
+        vectorstore=arthur_vectorstore,
+        combine_docs_chain=chat_response_generator,
+        question_generator=question_generator,
+        callback_manager=manager,
+        return_source_documents=True)
+
+    os.environ["OPENAI_API_KEY"] = ""
+    return agent
+
+
+def get_source_doc(output):
+    sources = output["source_documents"]
+    assert len(sources) > 0
+    source_document = sources[0]
+    html_filename = source_document.metadata['source']
+    source_doc_link = html_filename.replace('files/', '')
+    source_doc_file = html_filename.replace('files/docs.arthur.ai/', '').replace('.html', '')
+    with open(f"files/arthur-docs-markdown/{source_doc_file}.md.txt", 'r') as f:
+        source_text = f.read()
+    return source_text, source_doc_link
+
+
+def log_inference(chat_history: List[List[str]], llm_feedback: int) -> None:
+    reference_data = pd.read_csv("files/reference_data.csv", index_col=None)
+    chat_text = []
+    for user_text, bot_text in chat_history:
+        bot_text = bot_text.replace("\n", "").replace("<br>", "")
+        chat_text.append(f"input:<{user_text}>,output:<{bot_text}>,")
+    reference_data.loc[len(reference_data)] = {"chat": "".join(chat_text), "llm_feedback": llm_feedback}
+    reference_data.to_csv("files/reference_data.csv", index=False)
+
+
+def chat(inp, history, agent):
+    history = history or []
+    result = agent({"question": inp, "chat_history": history})
+    chat_result = result["answer"]
+    source_doc, source_link = get_source_doc(result)
+    response = ""
+    for word in chat_result.split(" "):
+        response += word + " "
+        yield history + [(inp, response)], history + [(inp, response)], source_doc, source_link
+
+
+def launch_ask_arthur(share=False):
+    with gr.Blocks() as demo:
+        with gr.Row():
+            gr.Markdown("<h1><center>Ask Arthur</center></h1><br><h7><center>"
+                        "This is an experimental document-retrieval question-answering system. When you enter a message"
+                        " into the chat, 1. we fetch a relevant page from the Arthur documentation (shown on the left) "
+                        "and 2. we prompt the Arthur LLM (shown on the right) to answer your question using the page. "
+                        "The LLM's answers are entirely unverified, but it can sometimes offer a helpful summary of a "
+                        "lot of information, or integrate information from multiple sources for you.</center></h7>")
+            with gr.Column():
+                openai_api_key_textbox = gr.Textbox(
+                    placeholder="Paste your OpenAI API key (sk-...)",
+                    show_label=False,
+                    lines=1,
+                    type="password",
+                )
+                submit_api_key_button = gr.Button(value="Register API Key", variant="secondary").style(full_width=False)
+        with gr.Row().style():
+            with gr.Column():
+                chatbot = gr.Chatbot(
+                    label="AskArthur chat history")
+                message = gr.Textbox(
+                    label="In the AskArthur chat, you can ask follow up questions or ask for clarifications!"
+                          "\nReload the demo to change the topic of conversation and refresh the language model.",
+                    placeholder="Enter your question here...",
+                    lines=1,
+                )
+                submit_message = gr.Button(value="Send", variant="secondary").style(full_width=False)
+                gr.Examples(label="Frequently asked questions about Arthur",
+                    examples=[
+                        "What default drift metrics does Arthur for deployed models?",
+                        "How do I integrate Arthur with my existing ML stack?",
+                        "How does Arthur monitor models without ground truth?",
+                        "Can you explain the following error message: ",
+                    ],
+                    inputs=message,
+                )
+                # feedback radio button
+                llm_feedback = gr.Radio(
+                    ["0","1","2"], value="0", label="How useful was this? (0 = bad, 1 = meh, 2 = good)"
+                )
+                submit_feedback_button = gr.Button("Submit feedback")
+            with gr.Column():
+                source_link = gr.Markdown()
+                source_page = gr.Markdown()
+
+        state = gr.State()
+        agent_state = gr.State()
+        submit_api_key_button.click(
+            get_langchain_agent,
+            inputs=[openai_api_key_textbox],
+            outputs=[agent_state],
+        )
+        submit_message.click(chat, inputs=[message, state, agent_state], outputs=[chatbot, state, source_page, source_link])
+        message.submit(chat, inputs=[message, state, agent_state], outputs=[chatbot, state, source_page, source_link])
+
+
+        submit_feedback_button.click(
+            log_inference,
+            [chatbot, llm_feedback],
+        )
+
+    demo.queue().launch(share=share)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--share", type=bool, required=False)
+    args = parser.parse_args()
+    launch_ask_arthur(share=args.share)