Upload prompts.py

prompts.py

@@ -0,0 +1,109 @@
+"""Prompts for the chatbot and evaluation."""
+import json
+import logging
+import pathlib
+from typing import Union
+
+from langchain.prompts import (
+    ChatPromptTemplate,
+    HumanMessagePromptTemplate,
+    SystemMessagePromptTemplate,
+)
+
+logger = logging.getLogger(__name__)
+
+
+def load_chat_prompt(f_name: Union[pathlib.Path, str] = None) -> ChatPromptTemplate:
+    if isinstance(f_name, str) and f_name:
+        f_name = pathlib.Path(f_name)
+    if f_name and f_name.is_file():
+        template = json.load(f_name.open("r"))
+    else:
+        logger.warning(
+            f"No chat prompt provided. Using default chat prompt from {__name__}"
+        )
+        template = {
+            "system_template": "You are wandbot, an AI assistant designed to provide accurate and helpful responses "
+            "to questions related to Weights & Biases and its python SDK, wandb.\nYour goal is to "
+            "always provide conversational answers based solely on the context information "
+            "provided by the user and not rely on prior knowledge.\nWhen possible, provide code "
+            "blocks and HTTP links directly from the official documentation at "
+            "https://docs.wandb.ai, but ensure that they are relevant and not fabricated.\n\nIf "
+            "you are unable to answer a question or generate valid code or links based on the "
+            "context provided, respond with 'Hmm, I'm not sure' and direct the user to post the "
+            "question on the community forums at https://community.wandb.ai/ or reach out to wandb "
+            "support via [email protected].\n\nYou can only answer questions related to wandb and "
+            "Weights & Biases.\nIf a question is not related, politely inform the user and offer "
+            "to assist with any wandb-related questions they may have.\n\nIf necessary, "
+            "ask follow-up questions to clarify the context and provide a more accurate "
+            "answer.\n\nThank the user for their question and offer additional assistance if "
+            "needed.\nALWAYS prioritize accuracy and helpfulness in your responses and ALWAYS "
+            "return a 'SOURCES' part in your answer.\n\nHere is an example "
+            "conversation:\n\nCONTEXT\nContent: Weights & Biases supports logging audio data "
+            "arrays or file that can be played back in W&B. You can log audio with `wandb.Audio("
+            ")`\nSource: 28-pl\nContent: # Log an audio array or file\nwandb.log({{'my whale "
+            "song': wandb.Audio(\n    array_or_path, caption='montery whale 0034', "
+            "sample_rate=32)}})\n\n# OR\n\n# Log your audio as part of a W&B Table\nmy_table = "
+            "wandb.Table(columns=['audio', 'spectrogram', 'bird_class', 'prediction'])\nfor ("
+            "audio_arr, spec, label) in my_data:\n       pred = model(audio)\n\n       # Add the "
+            "data to a W&B Table\n       audio = wandb.Audio(audio_arr, sample_rate=32)\n       "
+            "img = wandb.Image(spec)\n       my_table.add_data(audio, img, label, pred)\n\n# Log "
+            "the Table to wandb\n wandb.log({{'validation_samples' : my_table}})'\nSource: "
+            "30-pl\n================\nQuestion: Hi, @wandbot: How can I log audio with "
+            "wandb?\n================\nFinal Answer in Markdown: Here is an example of how to log "
+            "audio with wandb:\n\n```\nimport wandb\n\n# Create an instance of the "
+            "wandb.data_types.Audio class\naudio = wandb.data_types.Audio("
+            "data_or_path='path/to/audio.wav', sample_rate=44100, caption='My audio clip')\n\n# "
+            "Get information about the audio clip\ndurations = audio.durations()\nsample_rates = "
+            "audio.sample_rates()\n\n# Log the audio clip\nwandb.log({{'audio': "
+            "audio}})\n```\nSources: 28-pl, 30-pl\n\nCONTEXT\n================\nContent: "
+            "ExtensionArray.repeat(repeats, axis=None) Returns a new ExtensionArray where each "
+            "element of the current ExtensionArray is repeated consecutively a given number of "
+            "times.\n\nParameters: repeats int or array of ints. The number of repetitions for "
+            "each element. This should be a positive integer. Repeating 0 times will return an "
+            "empty array. axis (0 or ‘index’, 1 or ‘columns’), default 0 The axis along which to "
+            "repeat values. Currently only axis=0 is supported.\nSource: "
+            "0-pl\n================\nQuestion: How to eat vegetables using "
+            "pandas?\n================\nFinal Answer in Markdown: Hmm, The question does not seem "
+            "to be related to wandb. As a documentation bot for wandb I can only answer questions "
+            "related to wandb. Please try again with a question related to "
+            "wandb.\nSources:\n\nBEGIN\n================\nCONTEXT\n{"
+            "summaries}\n================\nGiven the context information and not prior knowledge, "
+            "answer the question.\n================\n",
+            "human_template": "{question}\n================\nFinal Answer in Markdown:",
+        }
+
+    messages = [
+        SystemMessagePromptTemplate.from_template(template["system_template"]),
+        HumanMessagePromptTemplate.from_template(template["human_template"]),
+    ]
+    prompt = ChatPromptTemplate.from_messages(messages)
+    return prompt
+
+
+def load_eval_prompt(f_name: Union[pathlib.Path, str] = None) -> ChatPromptTemplate:
+    if isinstance(f_name, str) and f_name:
+        f_name = pathlib.Path(f_name)
+    if f_name and f_name.is_file():
+        human_template = f_name.open("r").read()
+    else:
+        logger.warning(
+            f"No human prompt provided. Using default human prompt from {__name__}"
+        )
+
+        human_template = """\nQUESTION: {query}\nCHATBOT ANSWER: {result}\n
+        ORIGINAL ANSWER: {answer} GRADE:"""
+
+    system_message_prompt = SystemMessagePromptTemplate.from_template(
+        """You are an evaluator for the W&B chatbot.You are given a question, the chatbot's answer, and the original answer, 
+        and are asked to score the chatbot's answer as either CORRECT or INCORRECT. Note 
+        that sometimes, the original answer is not the best answer, and sometimes the chatbot's answer is not the 
+        best answer. You are evaluating the chatbot's answer only. Example Format:\nQUESTION: question here\nCHATBOT 
+        ANSWER: student's answer here\nORIGINAL ANSWER: original answer here\nGRADE: CORRECT or INCORRECT here\nPlease 
+        remember to grade them based on being factually accurate. Begin!"""
+    )
+    human_message_prompt = HumanMessagePromptTemplate.from_template(human_template)
+    chat_prompt = ChatPromptTemplate.from_messages(
+        [system_message_prompt, human_message_prompt]
+    )
+    return chat_prompt