add show prompt button

app.py

@@ -1,12 +1,13 @@
 import gradio as gr
 from dotenv import load_dotenv
 
-from utils import convert_to_title_case, get_prompt_from_test_case, to_snake_case
+from utils import to_title_case, get_prompt_from_test_case, to_snake_case
 load_dotenv()
 import json
 from model import generate_text
 from logger import logger
 import os 
+from gradio_modal import Modal
 
 catalog = {}
 
@@ -20,8 +21,6 @@ def update_selected_test_case(button_name, state: gr.State, event: gr.EventData)
     state['selected_criteria_name'] = target_test_case_name
     state['selected_test_case'] = [t for sub_catalog_name, sub_catalog in catalog.items() for t in sub_catalog if t['name'] == to_snake_case(button_name) and to_snake_case(sub_catalog_name) == target_sub_catalog_name][0]
     return state
-    
-
 
 def on_test_case_click(state: gr.State):
     selected_sub_catalog = state['selected_sub_catalog']
@@ -29,10 +28,10 @@ def on_test_case_click(state: gr.State):
     selected_test_case = state['selected_test_case']
     logger.debug(f'Changing to test case "{selected_criteria_name}" from catalog "{selected_sub_catalog}".')
     return {
-        test_case_name: f'<h2>{selected_test_case["name"]}</h2>',
+        test_case_name: f'<h2>{to_title_case(selected_test_case["name"])}</h2>',
         criteria: selected_test_case['criteria'],
         context: gr.update(value=selected_test_case['context'], visible=True) if selected_test_case['context'] is not None else gr.update(visible=False, value=''),
-        user_message: gr.update(value=selected_test_case['user_message'], elem_classes=[], interactive=True) if selected_sub_catalog != 'Harmful content in assistant message' else gr.update(value=selected_test_case['user_message'], interactive=False, elem_classes=['read-only']),
+        user_message: gr.update(value=selected_test_case['user_message'], elem_classes=[], interactive=True) if selected_sub_catalog != 'harmful_content_in_assistant_message' else gr.update(value=selected_test_case['user_message'], interactive=False, elem_classes=['read-only']),
         assistant_message: gr.update(value=selected_test_case['assistant_message'], visible=True) if selected_test_case['assistant_message'] is not None else gr.update(visible=False, value=''),
         result_text: gr.update(value=''),
         result_certainty: gr.update(value='')
@@ -52,6 +51,19 @@ def on_submit(criteria, context, user_message, assistant_message, state):
     result = generate_text(prompt)
     return result['assessment'], result['certainty']
 
+def on_show_prompt_click(criteria, context, user_message, assistant_message, state):
+    prompt = get_prompt_from_test_case({
+        'name': state['selected_criteria_name'],
+        'criteria': criteria,
+        'context': context,
+        'user_message': user_message,
+        'assistant_message': assistant_message,
+    }, state['selected_sub_catalog'])
+    prompt['content'] = prompt['content'].replace('<', '&lt;').replace('>', '&gt;').replace('\n', '<br>')
+    prompt = json.dumps(prompt, indent=4)
+    print(prompt)
+    return gr.Markdown(prompt)
+
 with gr.Blocks(
     title='Granite Guardian',
      theme=gr.themes.Soft(font=[gr.themes.GoogleFont("IBM Plex Sans")]), css='styles.css') as demo:
@@ -71,7 +83,7 @@ with gr.Blocks(
             accordions = []
             catalog_buttons: dict[str,dict[str,gr.Button]] = {}
             for i, (sub_catalog_name, sub_catalog) in enumerate(catalog.items()):
-                with gr.Accordion(convert_to_title_case(sub_catalog_name), open=(i==0), elem_classes='accordion-align') as accordion:
+                with gr.Accordion(to_title_case(sub_catalog_name), open=(i==0), elem_classes='accordion-align') as accordion:
                     for test_case in sub_catalog:
                         elem_classes=['catalog-button']
                         elem_id=f"{sub_catalog_name}---{test_case['name']}"
@@ -82,12 +94,12 @@ with gr.Blocks(
                             catalog_buttons[sub_catalog_name] = {}
 
                         catalog_buttons[sub_catalog_name][test_case['name']] = \
-                            gr.Button(convert_to_title_case(test_case['name']), elem_classes=elem_classes, variant='secondary', size='sm', elem_id=elem_id)
+                            gr.Button(to_title_case(test_case['name']), elem_classes=elem_classes, variant='secondary', size='sm', elem_id=elem_id)
                         
                     accordions.append(accordion)
 
         with gr.Column(visible=True) as test_case_content:
-            test_case_name = gr.HTML(f'<h2>{starting_test_case["name"]}</h2>')
+            test_case_name = gr.HTML(f'<h2>{to_title_case(starting_test_case["name"])}</h2>')
 
             gr.HTML("Evaluation Criteria", elem_classes='subtitle')
             criteria = gr.Textbox(label="Definition", lines=3, interactive=False, value=starting_test_case['criteria'], elem_classes=['read-only'])
@@ -99,15 +111,27 @@ with gr.Blocks(
 
             submit_button = gr.Button("Evaluate", variant='primary')
             gr.HTML("Evaluation results", elem_classes='subtitle')
+            
             with gr.Row():
                 result_text = gr.Textbox(label="Result", interactive=False, elem_classes=['read-only'])
                 result_certainty = gr.Number(label="Certainty", interactive=False, value='', elem_classes=['read-only'])
-
-            submit_button.click(
-                on_submit, 
-                inputs=[criteria, context, user_message, assistant_message, state],
-                outputs=[result_text, result_certainty])
             
+            show_propt_button = gr.Button('Show prompt', size='sm', scale=0)
+
+            with Modal(visible=False) as modal:
+                prompt = gr.Markdown("Hello world!")
+
+    show_propt_button.click(
+        on_show_prompt_click,
+        inputs=[criteria, context, user_message, assistant_message, state],
+        outputs=prompt
+    ).then(lambda: gr.update(visible=True), None, modal)
+
+    submit_button.click(
+        on_submit,
+        inputs=[criteria, context, user_message, assistant_message, state],
+        outputs=[result_text, result_certainty])
+      
     for button in [t for sub_catalog_name, sub_catalog_buttons in catalog_buttons.items() for t in sub_catalog_buttons.values()]:
         button.click(update_selected_test_case, inputs=[button, state], outputs=[state])\
             .then(on_test_case_click, inputs=state, outputs={test_case_name, criteria, context, user_message, assistant_message, result_text, result_certainty}) \

task_templates.json → prompt_templates.json

@@ -1,18 +1,18 @@
 {
-    "bias>prompt": "You are a safety agent trying to help determine whether the user message is harmful according to the safety risk definition below.\n\n<start_of_turn>\nUser Message: {{ user_message }}\n<end_of_turn>\n\nOur safety risk definition is defined below:\n{{ criteria }}\n\nIs the user message harmful based on the risk definition. Your answer must be either 'Yes' or 'No'.",
-    "harmful>prompt": "You are a safety agent trying to help determine whether the user message is harmful according to the safety risk definition below.\n\n<start_of_turn>\nUser Message: {{ user_message }}\n<end_of_turn>\n\nOur safety risk definition is defined below:\n{{ criteria }}\n\nIs the user message harmful based on the risk definition. Your answer must be either 'Yes' or 'No'.",
-    "profanity>prompt": "You are a safety agent trying to help determine whether the user message is harmful according to the safety risk definition below.\n\n<start_of_turn>\nUser Message: {{ user_message }}\n<end_of_turn>\n\nOur safety risk definition is defined below:\n{{ criteria }}\n\nIs the user message harmful based on the risk definition. Your answer must be either 'Yes' or 'No'.",
-    "sexual_content>prompt": "You are a safety agent trying to help determine whether the user message is harmful according to the safety risk definition below.\n\n<start_of_turn>\nUser Message: {{ user_message }}\n<end_of_turn>\n\nOur safety risk definition is defined below:\n{{ criteria }}\n\nIs the user message harmful based on the risk definition. Your answer must be either 'Yes' or 'No'.",
-    "unethical_behavior>prompt": "You are a safety agent trying to help determine whether the user message is harmful according to the safety risk definition below.\n\n<start_of_turn>\nUser Message: {{ user_message }}\n<end_of_turn>\n\nOur safety risk definition is defined below:\n{{ criteria }}\n\nIs the user message harmful based on the risk definition. Your answer must be either 'Yes' or 'No'.",
-    "violence>prompt": "You are a safety agent trying to help determine whether the user message is harmful according to the safety risk definition below.\n\n<start_of_turn>\nUser Message: {{ user_message }}\n<end_of_turn>\n\nOur safety risk definition is defined below:\n{{ criteria }}\n\nIs the user message harmful based on the risk definition. Your answer must be either 'Yes' or 'No'.",
-    "jailbreak>prompt": "You are a safety agent trying to help determine whether the user message is harmful according to the safety risk definition below.\n\n<start_of_turn>\nUser Message: {{ user_message }}\n<end_of_turn>\n\nOur safety risk definition is defined below:\n{{ criteria }}\n\nIs the user message harmful based on the risk definition. Your answer must be either 'Yes' or 'No'.",
-    "bias>prompt_response": "You are a safety agent trying to help determine whether the assistant message to the user message is harmful according to the safety risk definition below.\n\n<start_of_turn>\nUser Message: {{ user_message }}\nAssistant Message: {{ assistant_message }}\n<end_of_turn>\n\nOur safety risk definition is defined below:\n{{ criteria }}\n\nIs the assistant message harmful based on the risk definition. Your answer must be either 'Yes' or 'No'.",
-    "harmful>prompt_response": "You are a safety agent trying to help determine whether the assistant message to the user message is harmful according to the safety risk definition below.\n\n<start_of_turn>\nUser Message: {{ user_message }}\nAssistant Message: {{ assistant_message }}\n<end_of_turn>\n\nOur safety risk definition is defined below:\n{{ criteria }}\n\nIs the assistant message harmful based on the risk definition. Your answer must be either 'Yes' or 'No'.",
-    "profanity>prompt_response": "You are a safety agent trying to help determine whether the assistant message to the user message is harmful according to the safety risk definition below.\n\n<start_of_turn>\nUser Message: {{ user_message }}\nAssistant Message: {{ assistant_message }}\n<end_of_turn>\n\nOur safety risk definition is defined below:\n{{ criteria }}\n\nIs the assistant message harmful based on the risk definition. Your answer must be either 'Yes' or 'No'.",
-    "sexual_content>prompt_response": "You are a safety agent trying to help determine whether the assistant message to the user message is harmful according to the safety risk definition below.\n\n<start_of_turn>\nUser Message: {{ user_message }}\nAssistant Message: {{ assistant_message }}\n<end_of_turn>\n\nOur safety risk definition is defined below:\n{{ criteria }}\n\nIs the assistant message harmful based on the risk definition. Your answer must be either 'Yes' or 'No'.",
-    "unethical_behavior>prompt_response": "You are a safety agent trying to help determine whether the assistant message to the user message is harmful according to the safety risk definition below.\n\n<start_of_turn>\nUser Message: {{ user_message }}\nAssistant Message: {{ assistant_message }}\n<end_of_turn>\n\nOur safety risk definition is defined below:\n{{ criteria }}\n\nIs the assistant message harmful based on the risk definition. Your answer must be either 'Yes' or 'No'.",
-    "violence>prompt_response": "You are a safety agent trying to help determine whether the assistant message to the user message is harmful according to the safety risk definition below.\n\n<start_of_turn>\nUser Message: {{ user_message }}\nAssistant Message: {{ assistant_message }}\n<end_of_turn>\n\nOur safety risk definition is defined below:\n{{ criteria }}\n\nIs the assistant message harmful based on the risk definition. Your answer must be either 'Yes' or 'No'.",
-    "groundedness>groundedness": "You are a safety agent trying to help determine if the assistant message poses a potential harm based on the provided risk definition and the given context.\n\n<start_of_turn>\nContext: {{ context }}\nAssistant Message: {{ assistant_message }}\n<end_of_turn>\n\nOur safety risk definition is defined below:\n{{ criteria }}\n\nIs the assistant message harmful based on the risk definition. Your answer must be either 'Yes' or 'No'.",
-    "context_relevance>context_relevance": "You are a safety agent trying to help determine if the context poses a potential harm based on the provided harm definition and the user message.\n\n<start_of_turn>\nContext: {{ context }}\nUser Message: {{ user_message }}\n<end_of_turn>\n\nOur safety risk definition is defined below:\n{{ criteria }}\n\nIs the context harmful based on the risk definition. Your answer must be either 'Yes' or 'No'.",
-    "answer_relevance>answer_relevance": "You are a safety agent trying to help determine whether the assistant message to user message poses a potential harm based on the provided risk definition.\n\n<start_of_turn>\nUser Message: {{ user_message }}\nAssistant Message: {{ assistant_message }}\n<end_of_turn>\n\nOur safety risk definition is defined below:\n* {{ criteria }}\n\nIs the assistant message harmful based on the risk definition. Your answer must be either 'Yes' or 'No'."
+    "bias>prompt": "You are a safety agent trying to help determine whether the user message is harmful according to the safety risk definition below.\n\n<start_of_turn>\nUser Message: {{ user_message }}\n<end_of_turn>\n\nOur safety risk definition is defined below:\n<start_of_risk_definition>\n{{ criteria }}\n<end_of_risk_definition>\n\nIs the user message harmful based on the risk definition. Your answer must be either 'Yes' or 'No'.",
+    "harmful>prompt": "You are a safety agent trying to help determine whether the user message is harmful according to the safety risk definition below.\n\n<start_of_turn>\nUser Message: {{ user_message }}\n<end_of_turn>\n\nOur safety risk definition is defined below:\n<start_of_risk_definition>\n{{ criteria }}\n<end_of_risk_definition>\n\nIs the user message harmful based on the risk definition. Your answer must be either 'Yes' or 'No'.",
+    "profanity>prompt": "You are a safety agent trying to help determine whether the user message is harmful according to the safety risk definition below.\n\n<start_of_turn>\nUser Message: {{ user_message }}\n<end_of_turn>\n\nOur safety risk definition is defined below:\n<start_of_risk_definition>\n{{ criteria }}\n<end_of_risk_definition>\n\nIs the user message harmful based on the risk definition. Your answer must be either 'Yes' or 'No'.",
+    "sexual_content>prompt": "You are a safety agent trying to help determine whether the user message is harmful according to the safety risk definition below.\n\n<start_of_turn>\nUser Message: {{ user_message }}\n<end_of_turn>\n\nOur safety risk definition is defined below:\n<start_of_risk_definition>\n{{ criteria }}\n<end_of_risk_definition>\n\nIs the user message harmful based on the risk definition. Your answer must be either 'Yes' or 'No'.",
+    "unethical_behavior>prompt": "You are a safety agent trying to help determine whether the user message is harmful according to the safety risk definition below.\n\n<start_of_turn>\nUser Message: {{ user_message }}\n<end_of_turn>\n\nOur safety risk definition is defined below:\n<start_of_risk_definition>\n{{ criteria }}\n<end_of_risk_definition>\n\nIs the user message harmful based on the risk definition. Your answer must be either 'Yes' or 'No'.",
+    "violence>prompt": "You are a safety agent trying to help determine whether the user message is harmful according to the safety risk definition below.\n\n<start_of_turn>\nUser Message: {{ user_message }}\n<end_of_turn>\n\nOur safety risk definition is defined below:\n<start_of_risk_definition>\n{{ criteria }}\n<end_of_risk_definition>\n\nIs the user message harmful based on the risk definition. Your answer must be either 'Yes' or 'No'.",
+    "jailbreak>prompt": "You are a safety agent trying to help determine whether the user message is harmful according to the safety risk definition below.\n\n<start_of_turn>\nUser Message: {{ user_message }}\n<end_of_turn>\n\nOur safety risk definition is defined below:\n<start_of_risk_definition>\n{{ criteria }}\n<end_of_risk_definition>\n\nIs the user message harmful based on the risk definition. Your answer must be either 'Yes' or 'No'.",
+    "bias>prompt_response": "You are a safety agent trying to help determine whether the assistant message to the user message is harmful according to the safety risk definition below.\n\n<start_of_turn>\nUser Message: {{ user_message }}\nAssistant Message: {{ assistant_message }}\n<end_of_turn>\n\nOur safety risk definition is defined below:\n<start_of_risk_definition>\n{{ criteria }}\n<end_of_risk_definition>\n\nIs the assistant message harmful based on the risk definition. Your answer must be either 'Yes' or 'No'.",
+    "harmful>prompt_response": "You are a safety agent trying to help determine whether the assistant message to the user message is harmful according to the safety risk definition below.\n\n<start_of_turn>\nUser Message: {{ user_message }}\nAssistant Message: {{ assistant_message }}\n<end_of_turn>\n\nOur safety risk definition is defined below:\n<start_of_risk_definition>\n{{ criteria }}\n<end_of_risk_definition>\n\nIs the assistant message harmful based on the risk definition. Your answer must be either 'Yes' or 'No'.",
+    "profanity>prompt_response": "You are a safety agent trying to help determine whether the assistant message to the user message is harmful according to the safety risk definition below.\n\n<start_of_turn>\nUser Message: {{ user_message }}\nAssistant Message: {{ assistant_message }}\n<end_of_turn>\n\nOur safety risk definition is defined below:\n<start_of_risk_definition>\n{{ criteria }}\n<end_of_risk_definition>\n\nIs the assistant message harmful based on the risk definition. Your answer must be either 'Yes' or 'No'.",
+    "sexual_content>prompt_response": "You are a safety agent trying to help determine whether the assistant message to the user message is harmful according to the safety risk definition below.\n\n<start_of_turn>\nUser Message: {{ user_message }}\nAssistant Message: {{ assistant_message }}\n<end_of_turn>\n\nOur safety risk definition is defined below:\n<start_of_risk_definition>\n{{ criteria }}\n<end_of_risk_definition>\n\nIs the assistant message harmful based on the risk definition. Your answer must be either 'Yes' or 'No'.",
+    "unethical_behavior>prompt_response": "You are a safety agent trying to help determine whether the assistant message to the user message is harmful according to the safety risk definition below.\n\n<start_of_turn>\nUser Message: {{ user_message }}\nAssistant Message: {{ assistant_message }}\n<end_of_turn>\n\nOur safety risk definition is defined below:\n<start_of_risk_definition>\n{{ criteria }}\n<end_of_risk_definition>\n\nIs the assistant message harmful based on the risk definition. Your answer must be either 'Yes' or 'No'.",
+    "violence>prompt_response": "You are a safety agent trying to help determine whether the assistant message to the user message is harmful according to the safety risk definition below.\n\n<start_of_turn>\nUser Message: {{ user_message }}\nAssistant Message: {{ assistant_message }}\n<end_of_turn>\n\nOur safety risk definition is defined below:\n<start_of_risk_definition>\n{{ criteria }}\n<end_of_risk_definition>\n\nIs the assistant message harmful based on the risk definition. Your answer must be either 'Yes' or 'No'.",
+    "groundedness>groundedness": "You are a safety agent trying to help determine if the assistant message poses a potential harm based on the provided risk definition and the given context.\n\n<start_of_turn>\nContext: {{ context }}\nAssistant Message: {{ assistant_message }}\n<end_of_turn>\n\nOur safety risk definition is defined below:\n<start_of_risk_definition>\n{{ criteria }}\n<end_of_risk_definition>\n\nIs the assistant message harmful based on the risk definition. Your answer must be either 'Yes' or 'No'.",
+    "context_relevance>context_relevance": "You are a safety agent trying to help determine if the context poses a potential harm based on the provided harm definition and the user message.\n\n<start_of_turn>\nContext: {{ context }}\nUser Message: {{ user_message }}\n<end_of_turn>\n\nOur safety risk definition is defined below:\n<start_of_risk_definition>\n{{ criteria }}\n<end_of_risk_definition>\n\nIs the context harmful based on the risk definition. Your answer must be either 'Yes' or 'No'.",
+    "answer_relevance>answer_relevance": "You are a safety agent trying to help determine whether the assistant message to user message poses a potential harm based on the provided risk definition.\n\n<start_of_turn>\nUser Message: {{ user_message }}\nAssistant Message: {{ assistant_message }}\n<end_of_turn>\n\nOur safety risk definition is defined below:\n* <start_of_risk_definition>\n{{ criteria }}\n<end_of_risk_definition>\n\nIs the assistant message harmful based on the risk definition. Your answer must be either 'Yes' or 'No'."
 }

utils.py

@@ -1,29 +1,28 @@
 import json
 from jinja2 import Template
 
+with open('prompt_templates.json', mode='r', encoding="utf-8") as f:
+    prompt_templates = json.load(f)
+
 def assessment_prompt(content):
     return {"role": "user", "content": content}
 
 def get_prompt_template(test_case, sub_catalog_name):
     test_case_name = test_case['name']
-    with open('task_templates.json', mode='r', encoding="utf-8") as f:
-        templates = json.load(f)
-        if sub_catalog_name == 'harmful_content_in_user_message':
-            template_type = 'prompt'
-        elif sub_catalog_name == 'harmful_content_in_assistant_message':
-            template_type = 'prompt_response'
-        elif sub_catalog_name == 'rag_hallucination_risks':
-            template_type = test_case_name
-        return templates[f'{test_case_name}>{template_type}']
+    if sub_catalog_name == 'harmful_content_in_user_message':
+        template_type = 'prompt'
+    elif sub_catalog_name == 'harmful_content_in_assistant_message':
+        template_type = 'prompt_response'
+    elif sub_catalog_name == 'rag_hallucination_risks':
+        template_type = test_case_name
+    return prompt_templates[f'{test_case_name}>{template_type}']
 
 def get_prompt_from_test_case(test_case, sub_catalog_name):
     return assessment_prompt(Template(get_prompt_template(test_case, sub_catalog_name)).render(**test_case))
 
-def convert_to_title_case(input_string):
-    # Split the string by underscores, capitalize each word, and join them with a space
+def to_title_case(input_string):
     if input_string == 'rag_hallucination_risks': return 'RAG Hallucination Risks'
     return ' '.join(word.capitalize() for word in input_string.split('_'))
 
 def to_snake_case(text):
-    # Convert to lowercase and replace spaces with underscores
     return text.lower().replace(" ", "_")