Synced repo using 'sync_with_huggingface' Github Action

.env.example

@@ -0,0 +1,3 @@
+TOGETHER_API_KEY=your_together_api_key_here
+ANTHROPIC_API_KEY=your_anthropic_api_key_here
+OPENAI_API_KEY=your_openai_api_key_here

app.py

@@ -1,468 +1,58 @@
-import json
-import re
-from datetime import datetime
-import hashlib
-import gradio as gr
-
-from gen_api_answer import (
-     atla_parse_model_response,
-    get_atla_response
-)
-
-from prompts import (
-    ATLA_PROMPT,
-    ATLA_PROMPT_WITH_REFERENCE
-)
-
-from random_sample_generation import (
-    get_random_human_ai_pair,
-    get_random_human_ai_ground_truth_pair,
-    generate_ai_response
-)   
-
-from utils import Vote
-
-from prompts import (
-    DEFAULT_EVAL_PROMPT,
-    DEFAULT_EVAL_PROMPT_EDITABLE,
-    FIXED_EVAL_SUFFIX,
-    DEFAULT_EVAL_CRITERIA
-)
-
-from common import CSS_STYLES, MAIN_TITLE, HOW_IT_WORKS
-
-
-# Load the model_data from JSONL
-def load_model_data():
-    model_data = {}
-    try:
-        with open("data/models.jsonl", "r") as f:
-            for line in f:
-                model = json.loads(line)
-                model_data[model["name"]] = {
-                    "organization": model["organization"],
-                    "license": model["license"],
-                    "api_model": model["api_model"],
-                }
-    except FileNotFoundError:
-        print("Warning: models.jsonl not found")
-        return {}
-    return model_data
-
-
-model_data = load_model_data()
-
-def store_vote_data(prompt, response_a, response_b, model_a, model_b, winner, judge_id):
-    prompt_value = prompt.value if hasattr(prompt, 'value') else prompt
-    
-    vote = Vote(
-        timestamp=datetime.now().isoformat(),
-        prompt=prompt_value,
-        response_a=response_a,
-        response_b=response_b,
-        model_a=model_a,
-        model_b=model_b,
-        winner=winner,
-        judge_id=judge_id,
-    )
-    add_vote(vote, db)
-
-
-def parse_variables(prompt):
-    # Extract variables enclosed in double curly braces
-    variables = re.findall(r"{{(.*?)}}", prompt)
-    # Remove duplicates while preserving order
-    seen = set()
-    variables = [
-        x.strip() for x in variables if not (x.strip() in seen or seen.add(x.strip()))
-    ]
-    return variables
-
-
-def get_final_prompt(eval_prompt, variable_values):
-    # Replace variables in the eval prompt with their values
-    for var, val in variable_values.items():
-        eval_prompt = eval_prompt.replace("{{" + var + "}}", val)
-    return eval_prompt
-
-
-
-def get_ip(request: gr.Request) -> str:
-    """Get and hash the IP address from the request."""
-    if "cf-connecting-ip" in request.headers:
-        ip = request.headers["cf-connecting-ip"]
-    elif "x-forwarded-for" in request.headers:
-        ip = request.headers["x-forwarded-for"]
-        if "," in ip:
-            ip = ip.split(",")[0]
-    else:
-        ip = request.client.host
-    
-    # Hash the IP address for privacy
-    return hashlib.sha256(ip.encode()).hexdigest()[:16]
-
-
-def get_vote_message(choice: str, model_a: str, model_b: str) -> tuple[str, str]:
-    """Generate appropriate message based on vote and model rankings.
-    Returns (title, message) tuple."""
-    # Get current rankings
-    voting_data = get_current_votes()
-    leaderboard = get_leaderboard(model_data, voting_data, show_preliminary=True)
-    rankings = get_model_rankings(leaderboard)
-    pos_a = rankings.get(model_a, 0)
-    pos_b = rankings.get(model_b, 0)
-    
-    if choice == "Tie":
-        return "It's a tie!", "Keep voting responsibly 🤗"
-    
-    # Check if vote aligns with leaderboard
-    if (choice == "A" and pos_a < pos_b) or (choice == "B" and pos_b < pos_a):
-        return "The favourite wins!", "Keep voting responsibly 🤗"
-    else:
-        return "The underdog wins!", "Keep voting responsibly 🤗"
-
-
-def populate_random_example(request: gr.Request, compatible_mode: bool):
-    """Generate a random human-AI conversation example and reset judge outputs."""
-    if compatible_mode:
-        human_msg, ai_msg, ground_truth_msg = get_random_human_ai_ground_truth_pair()
-    else:
-        human_msg, ai_msg = get_random_human_ai_pair()
-        ground_truth_msg = ""
-    
-    return [
-        gr.update(value=human_msg),
-        gr.update(value=ai_msg),
-        gr.update(value="🎲", variant="secondary"),
-        gr.update(value=""),  # Clear score
-        gr.update(value=""),  # Clear critique
-        gr.update(value=ground_truth_msg, visible=compatible_mode),  # Set ground truth and visibility
-    ]
-
-
-with gr.Blocks(theme="default", css=CSS_STYLES) as demo:
-    gr.Markdown(MAIN_TITLE)
-    gr.Markdown(HOW_IT_WORKS)
-    
-    # Hidden eval prompt that will always contain DEFAULT_EVAL_PROMPT
-    eval_prompt = gr.Textbox(
-        value=DEFAULT_EVAL_PROMPT,
-        visible=False
-    )
-
-    with gr.Tabs():
-        with gr.TabItem("Playground"):
-            with gr.Row():
-                # Left side - Input section
-                with gr.Column(scale=1):
-                    with gr.Group():
-                        human_input = gr.TextArea(
-                            label="👩 User Input",
-                            lines=5,
-                            placeholder="Enter the human message here..."
-                        )
-                        with gr.Row():
-                            generate_btn = gr.Button(
-                                "Generate AI Response",
-                                size="sm",
-                                interactive=False
-                            )
-                        
-                        ai_response = gr.TextArea(
-                            label="🤖 AI Response", 
-                            lines=10,
-                            placeholder="Enter the AI response here..."
-                        )
-                        
-                        # Ground truth response (initially hidden)
-                        ground_truth = gr.TextArea(
-                            label="🎯 Ground truth response",
-                            lines=10,
-                            placeholder="Enter the ground truth response here...",
-                            visible=False
-                        )
-                        
-                    with gr.Row():
-                        random_btn = gr.Button("🎲", scale=2)
-                        send_btn = gr.Button(
-                            value="Run evaluation",
-                            variant="primary",
-                            size="lg",
-                            scale=8
-                        )
-
-                # Right side - Model outputs
-                with gr.Column(scale=1):
-                    gr.Markdown("### 👩‍⚖️ Selene-Mini Evaluation")
-                    with gr.Group():
-                        with gr.Row():
-                            score = gr.Textbox(label="Score", lines=1, interactive=False)
-                        critique = gr.TextArea(label="Critique", lines=12, interactive=False)
-                
-            gr.Markdown("<br>")
-            
+# sandbox_runner.py
 
-            # Replace the "Edit Judge Prompt" Accordion section with:
-            with gr.Accordion("📝 Edit Judge Prompt", open=False) as prompt_accordion:
-                gr.Markdown("<br>")
-                use_reference_toggle = gr.Checkbox(
-                    label="Use a reference response",
-                    value=False
-                )
-                
-                # Hide the default prompt editor
-                with gr.Column(visible=False) as default_prompt_editor:
-                    eval_prompt_editable = gr.TextArea(
-                        value=DEFAULT_EVAL_PROMPT_EDITABLE,
-                        label="Evaluation Criteria",
-                        lines=12
-                    )
-
-                    with gr.Row(visible=False) as edit_buttons_row:
-                        cancel_prompt_btn = gr.Button("Cancel")
-                        save_prompt_btn = gr.Button("Save", variant="primary")
-                    gr.Markdown("*The sample being evaluated is always appended as:*")
-                    gr.Markdown(f"```{FIXED_EVAL_SUFFIX}")
+import gradio as gr
+from data_handler import upload_test_data
+from criteria_handler import select_evaluation_criteria
+from model_handler import select_evaluators
+from score_handler import handle_analysis
+from random_sample_tab import random_sample_tab
+
+def run_sandbox():
+    with gr.Blocks(css="""
+    .truncate_cells table {
+        table-layout: fixed !important;
+        width: 100% !important;
+    }
+    .truncate_cells table td,
+    .truncate_cells table th {
+        white-space: nowrap !important;
+        overflow: hidden !important;
+        text-overflow: ellipsis !important;
+        max-width: 200px !important;
+        text-align: left !important;
+        vertical-align: top !important;
+    }
+    """) as demo:
+        gr.Markdown("# Selene Playground")
+        gr.Markdown("Try running evals with Selene and Selene-Mini in this playground! The Selene-Mini model card can be found [here](https://huggingface.co/AtlaAI/Selene-1-Mini-Llama-3.1-8B)")
+        with gr.Tabs():
+            # Random samples tab
+            random_sample_tab()
+
+            # Sandbox tab
+            with gr.TabItem("Custom dataset"):
+                # Initialize state object to track the DataFrame
+                df_state = gr.State(value=None)
+                # Initialize state object to track the prompt
+                prompt_state = gr.State(value=None)
+                # Initialize the evaluation_complete flag
+                evaluation_complete = gr.State(value=None)
+
+                # Data upload
+                data_upload_group, df_state = upload_test_data(df_state)
                 
-                # Show the compatible mode editor
-                with gr.Column(visible=True) as compatible_prompt_editor:
-                    eval_criteria_text = gr.TextArea(
-                        label="Evaluation Criteria",
-                        lines=12,
-                        value=DEFAULT_EVAL_CRITERIA,
-                        placeholder="Enter the complete evaluation criteria and scoring rubric..."
-                    )
-
-    # Define state variables for model tracking
-    model_a_state = gr.State()
-    model_b_state = gr.State()
-    final_prompt_state = gr.State()
-    eval_prompt_previous = gr.State(value=DEFAULT_EVAL_PROMPT_EDITABLE)  # Initialize with default value
-    is_editing = gr.State(False)  # Track editing state
-    compatible_mode_state = gr.State(False)  # Track compatible mode state
-
-    # Update model names after responses are generated
-    def update_model_names(model_a, model_b):
-        return gr.update(value=f"*Model: {model_a}*"), gr.update(
-            value=f"*Model: {model_b}*"
-        )
-
-    # Store the last submitted prompt and variables for comparison
-    last_submission = gr.State({})
-
-    # Add handlers for save/cancel buttons
-    def save_prompt(new_prompt, previous_prompt):
-        return [
-            gr.update(value=new_prompt),  # Update the prompt
-            new_prompt,  # Update the previous prompt state
-            gr.update(visible=False)  # Hide the buttons
-        ]
-
-    def cancel_prompt(previous_prompt):
-        return [
-            gr.update(value=previous_prompt),  # Revert to previous prompt
-            previous_prompt,  # Keep the previous prompt state
-            gr.update(visible=False)  # Hide the buttons
-        ]
-
-    def show_edit_buttons(current_value, previous_value):
-        # Show buttons only if the current value differs from the previous value
-        return gr.update(visible=current_value != previous_value)
-
-    # Add handlers for save/cancel buttons and prompt changes
-    save_prompt_btn.click(
-        fn=save_prompt,
-        inputs=[eval_prompt_editable, eval_prompt_previous],
-        outputs=[eval_prompt_editable, eval_prompt_previous, edit_buttons_row]
-    )
-
-    cancel_prompt_btn.click(
-        fn=cancel_prompt,
-        inputs=[eval_prompt_previous],
-        outputs=[eval_prompt_editable, eval_prompt_previous, edit_buttons_row]
-    )
-
-    eval_prompt_editable.change(
-        fn=show_edit_buttons,
-        inputs=[eval_prompt_editable, eval_prompt_previous],
-        outputs=edit_buttons_row
-    )
-
-    # Function to toggle visibility based on compatible mode
-    def toggle_use_reference(checked):
-        if checked:
-            human_msg, ai_msg, ground_truth_msg = get_random_human_ai_ground_truth_pair()
-            return {
-                ground_truth: gr.update(visible=True, value=ground_truth_msg),
-                human_input: gr.update(value=human_msg),
-                ai_response: gr.update(value=ai_msg),
-                score: gr.update(value=""),
-                critique: gr.update(value=""),
-                random_btn: gr.update(value="🎲", variant="secondary"),
-            }
-        else:
-            return {
-                ground_truth: gr.update(visible=False)
-            }
-
-    # Update the change handler to include all necessary outputs
-    use_reference_toggle.change(
-        fn=toggle_use_reference,
-        inputs=[use_reference_toggle],
-        outputs=[
-            ground_truth,
-            human_input,
-            ai_response,
-            score,
-            critique,
-            random_btn,
-        ]
-    )
-
-    # Add a new state variable to track first game
-    first_game_state = gr.State(True)  # Initialize as True
-
-    # Update the submit function to parse the evaluation criteria
-    def submit_and_store(
-        use_reference,
-        eval_criteria_text,
-        human_input,
-        ai_response,
-        ground_truth_input,
-    ):
-        # Build prompt data dictionary
-        prompt_data = {
-            'human_input': human_input,
-            'ai_response': ai_response,
-            'ground_truth_input': ground_truth_input if use_reference else '',
-            'eval_criteria': eval_criteria_text,
-        }
-
-        # Get base prompt based on whether reference is used
-        base_prompt = ATLA_PROMPT_WITH_REFERENCE if use_reference else ATLA_PROMPT
-
-        # Format the prompt
-        final_prompt = base_prompt.format(
-            human_input=prompt_data['human_input'],
-            ai_response=prompt_data['ai_response'],
-            ground_truth_input=prompt_data['ground_truth_input'],
-            eval_criteria=prompt_data['eval_criteria']
-        )
-
-        # Get response from Atla
-        response = get_atla_response(
-            model_name="AtlaAI/Selene-1-Mini-Llama-3.1-8B",
-            prompt=final_prompt,
-            max_tokens=500,
-            temperature=0.01
-        )
-
-        # Parse the response
-        score, critique = atla_parse_model_response(response)
-
-        return [
-            score,
-            critique,
-            gr.update(value="Regenerate evaluation", variant="secondary", interactive=True),
-            gr.update(value="🎲"),
-        ]
-
-    # Update the click handler to use False for is_first_game after first submission
-    def create_submit_handler():
-        first_game = True
-        
-        def handler(*args):
-            nonlocal first_game
-            result = submit_and_store(*args)
-            first_game = False  # Set to False after first submission
-            return result
-        
-        return handler
-
-    # Update the send_btn click handler
-    send_btn.click(
-        fn=submit_and_store,
-        inputs=[
-            use_reference_toggle,
-            eval_criteria_text,
-            human_input,
-            ai_response,
-            ground_truth,
-        ],
-        outputs=[
-            score,
-            critique,
-            send_btn,
-            random_btn,
-        ],
-    )
-
-    # Add random button handler
-    random_btn.click(
-        fn=populate_random_example,
-        inputs=[use_reference_toggle],
-        outputs=[
-            human_input,
-            ai_response,
-            random_btn,
-            score,
-            critique,
-            ground_truth,
-        ]
-    )
-
-    # Add input change handlers
-    def handle_input_change():
-        """Reset UI state when inputs are changed"""
-        return [
-            gr.update(value="Run evaluation", variant="primary"),  # send_btn
-            gr.update(value="🎲", variant="secondary"),  # random_btn
-        ]
-
-    # Update the change handlers for inputs
-    human_input.change(
-        fn=handle_input_change,
-        inputs=[],
-        outputs=[send_btn, random_btn]
-    )
-
-    ai_response.change(
-        fn=handle_input_change,
-        inputs=[],
-        outputs=[send_btn, random_btn]
-    )
+                # Criteria selection
+                criteria_group, df_state, prompt_state, save_prompt_button = \
+                    select_evaluation_criteria(data_upload_group, df_state, prompt_state)
 
-    generate_btn.click(
-        fn=lambda msg: (
-            generate_ai_response(msg)[0],  # Only take the response text
-            gr.update(
-                value="Generate AI Response",  # Keep the label
-                interactive=False  # Disable the button
-            )
-        ),
-        inputs=[human_input],
-        outputs=[ai_response, generate_btn]
-    )
+                # Models selection
+                model_selection_group, df_state, analyze_results_button = \
+                    select_evaluators(criteria_group, df_state, prompt_state, save_prompt_button)
 
-    human_input.change(
-        fn=lambda x: gr.update(interactive=bool(x.strip())),
-        inputs=[human_input],
-        outputs=[generate_btn]
-    )
+                # Result analysis
+                handle_analysis(df_state, model_selection_group, analyze_results_button)
 
-    # Update the demo.load to include the random example population
-    demo.load(
-        fn=lambda: populate_random_example(None, False),  # Pass False for initial compatible_mode
-        inputs=[],
-        outputs=[
-            human_input,
-            ai_response,
-            random_btn,
-            score,
-            critique,
-            ground_truth,
-        ]
-    )
+    demo.launch()
 
 if __name__ == "__main__":
-    demo.launch()
+    run_sandbox()

common.py

@@ -1,20 +1,18 @@
 # Page Headers
-MAIN_TITLE = "# Selene 1 Mini Playground"
+MAIN_TITLE = "# Judge Arena - Free LLM Evals to test your GenAI application"
 
 # How it works section
 HOW_IT_WORKS = """
-Try running evals with Selene-Mini in this playground! Our HF model card can be found [here](https://huggingface.co/AtlaAI/Selene-1-Mini-Llama-3.1-8B).
+- **Run any form of evaluation:** from simple hallucination detection to qualitative interpretations
+- **Evaluate anything:** coding, analysis, creative writing, math, or general knowledge
 """
 
 BATTLE_RULES = """
-## 🤺 Choose the winner
-1. Define your scoring criteria in the **Evaluator Prompt** 
-2. Add a test case to the **Sample to evaluate**
-3. Test the evaluators & vote for the model that best aligns with your judgement!
-\n
-Variables defined in your prompt with {{double curly braces}} map to input fields under **Sample to evaluate**.
-
-<br>
+## 🤺 Battle Rules:
+- Both AIs stay anonymous - if either reveals its identity, the duel is void
+- Choose the LLM judge that most aligns with your judgement
+- If both score the same - choose the critique that you prefer more!
+<br><br>
 """
 
 # CSS Styles
@@ -40,13 +38,40 @@ CSS_STYLES = """
 
 # Default Eval Prompt
 EVAL_DESCRIPTION = """
-## 📝 Tips
+## 📝 Instructions
 **Precise evaluation criteria leads to more consistent and reliable judgments.** A good evaluation prompt should include the following elements:
 - Evaluation criteria
 - Scoring rubric 
-- Examples (Optional)
+- (Optional) Examples\n
+
+**Any variables you define in your prompt using {{double curly braces}} will automatically map to the corresponding input fields under "Sample to evaluate" section on the right.**
+
+<br><br>
 """
 
+DEFAULT_EVAL_PROMPT = """You are assessing a chat bot response to a user's input based on [INSERT CRITERIA]
+
+Score:
+A score of 1 means that the response's answer meets all of the evaluation criteria.
+A score of 0 means that the response's answer does not meet all of the evaluation criteria.
+
+Here is the data:
+[BEGIN DATA]
+***
+[User Query]: {{input}}
+***
+[Response]: {{response}}
+***
+[END DATA]"""
+
+# Default Variable Values
+DEFAULT_INPUT = """Which of these animals is least likely to be found in a rainforest?"
+A) Jaguar
+B) Toucan
+C) Polar Bear
+D) Sloth"""
+DEFAULT_RESPONSE = "C) Polar Bear"
+
 # Voting Section Header
 VOTING_HEADER = """
 # Start Voting Now
@@ -54,7 +79,7 @@ VOTING_HEADER = """
 
 # Acknowledgements
 ACKNOWLEDGEMENTS = """
-<br><br>
+<br><br><br>
 # Acknowledgements
 
 We thank [LMSYS Org](https://lmsys.org/) for their hard work on the Chatbot Arena and fully credit them for the inspiration to build this.
@@ -68,50 +93,55 @@ POLICY_CONTENT = """
 
 Atla is an applied research organization that trains models as evaluators to capture human preferences. We're a team of researchers, engineers, and operational leaders, with experience spanning a variety of disciplines, all working together to build reliable and understandable AI systems. Our research is informed by our experiences conducting AI safety research at the UK AI Task Force, OpenAI and the Stanford Existential Risks Initiative.
 <br><br>
-# [Our Mission](https://www.atla-ai.com/company)
+# Our Mission
 
-By creating advanced evaluation models, we enable AI developers to identify and fix risks, leading to safer, more reliable AI that can be trusted and widely used. Our aim is to surpass the current state-of-the-art evaluation methods by training models specifically for evaluation. AIs will probably become very powerful, and perform tasks that are difficult for us to verify. We want to enable humans to oversee AI systems that are solving tasks too difficult for humans to evaluate. 
-Read more about [our approach to scalable oversight](https://www.atla-ai.com/post/scaling-alignment) on our blog.
+By creating advanced evaluation models, we enable AI developers to identify and fix risks, leading to safer, more reliable AI that can be trusted and widely used. Our aim is to surpass the current state-of-the-art evaluation methods by training models specifically for evaluation. AIs will probably become very powerful, and perform tasks that are difficult for us to verify. We want to enable humans to oversee AI systems that are solving tasks too difficult for humans to evaluate. We have written more about [our approach to scalable oversight](https://www.atla-ai.com/post/scaling-alignment) on our blog.
 <br><br>
 # Judge Arena Policy
 
 ## Overview
 
-Judge Arena is an open-source platform dedicated to determining which models make the best judges. Users can run evals and assess anonymized responses from two competing model judges, choosing the better judgement or declaring a tie. This policy outlines our commitments to maintain a fair and open environment :)
+Judge Arena is an open-source platform dedicated to improving the standard of evaluation of generative AI models in their role as judges. Users can run evals and assess anonymized responses from two competing model judges, choosing the better judgement or declaring a tie. This policy outlines our commitments to maintain a fair, open, and collaborative environment :)
 
 ## Transparency
 
 - **Open-Source**: Judge Arena's code is open-source and available on GitHub. We encourage contributions from the community and anyone can replicate or modify the platform to suit their needs. We use proprietary model provider APIs where provided and Together AI's API to serve leading open-source models.
-- **Methodology**: All processes related to model evaluation, rating calculations, and model selection are openly documented. 
+- **Methodology**: All processes related to model evaluation, rating calculations, and model selection are openly documented. We'd like to ensure that our ranking system is understandable and reproducible by others!
 - **Data Sharing**: Periodically, we'll share 20% of the collected evaluation data with the community. The data collected from Judge Arena is restricted to an anonymized user ID, the final prompt sent, the model responses, the user vote, and the timestamp.
 
 ## Model Inclusion Criteria
 
 Judge Arena is specifically designed to assess AI models that function as evaluators (a.k.a judges). This includes but is not limited to powerful general-purpose models and the latest language models designed for evaluation tasks. Models are eligible for inclusion if they meet the following criteria:
 
-- **Judge Capability**: The model should possess the ability to score AND critique other models' outputs effectively.
-- **Promptable:** The model must be promptable to be evaluate in different scoring formats, for different criteria.
+- **Judge Capability**: The model should possess the ability to score AND critique responses, content, or other models' outputs effectively.
+- **Adaptable:** The model must be prompt-able to be evaluate in different scoring formats, for different criteria.
 - **Accessibility**:
    - **Public API Access**: Models accessible through public APIs without restrictive barriers.
    - **Open-Source Models**: Models with publicly available weights that can be downloaded and run by the community.
 
 ## Leaderboard Management
 
-- **ELO Ranking System**: Models are ranked on a public leaderboard based on aggregated user evaluations. We use an ELO rating system to rank AI judges on the public leaderboard. Each model begins with an initial rating of 1200, and we use a K-factor of 32 to determine the maximum rating adjustment after each evaluation.
+- **ELO Ranking System**: Models are ranked on a public leaderboard based on aggregated user evaluations. We use an ELO rating system to rank AI judges on the public leaderboard. Each model begins with an initial rating of 1500 (as is used by the International Chess Federation), and we use a K-factor of 32 to determine the maximum rating adjustment after each evaluation.
 - **Minimum Period**: Listed models remain accessible on Judge Arena for a minimum period of two weeks so they can be comprehensively evaluated.
 - **Deprecation Policy**: Models may be removed from the leaderboard if they become inaccessible or are no longer publicly available.
 
-*This policy might be updated to reflect changes in our practices or in response to community feedback.*
-<br><br>
+This policy might be updated to reflect changes in our practices or in response to community feedback.
+
 # FAQ
 
 **Isn't this the same as Chatbot Arena?**
 
 We are big fans of what the LMSYS team have done with Chatbot Arena and fully credit them for the inspiration to develop this. We were looking for a dynamic leaderboard that graded on AI judge capabilities and didn't manage to find one, so we created Judge Arena. This UI is designed especially for evals; to match the format of the model-based eval prompts that you would use in your LLM evaluation / monitoring tool.
 
+**What are the Evaluator Prompt Templates based on?**
+
+As a quick start, we've set up templates that cover the most popular evaluation metrics out there on LLM evaluation / monitoring tools, often known as 'base metrics'. The data samples used in these were randomly picked from popular datasets from academia - [ARC](https://huggingface.co/datasets/allenai/ai2_arc), [Preference Collection](https://huggingface.co/datasets/prometheus-eval/Preference-Collection), [RewardBench](https://huggingface.co/datasets/allenai/reward-bench), [RAGTruth](https://arxiv.org/abs/2401.00396).
+
+These templates are designed as a starting point to showcase how to interact with the Judge Arena, especially for those less familiar with using LLM judges.
+
 **Why should I trust this leaderboard?**
 
-We have listed out our efforts to be fully transparent in the policies above. All of the code for this leaderboard is open-source and can be found on our [Github](https://github.com/atla-ai/judge-arena). Check out our [blog](https://www.atla-ai.com/blog) to stay up to date as we analyse the results from the leaderboard.
+We have listed out our efforts to be fully transparent in the policies above. All of the code for this leaderboard is open-source and can be found on our [Github](https://github.com/atla-ai/judge-arena).
 
 **Who funds this effort?**
 
@@ -122,5 +152,4 @@ Atla currently funds this out of our own pocket. We are looking for API credits
 We are training a general-purpose evaluator that you will soon be able to run in this Judge Arena. Our next step will be to open-source a powerful model that the community can use to run fast and accurate evaluations.
 <br><br>
 # Get in touch
-We’d love to hear your feedback! For general feature requests or to submit / suggest new models to add to the arena, please open up a discussion in the [community](https://huggingface.co/spaces/AtlaAI/judge-arena/discussions) tab. You can also contact us directly on [X](https://x.com/Atla_AI) or [Discord](https://discord.gg/yNpUAMqs).
-\nPlease file any issues on our [Github](https://github.com/atla-ai/judge-arena)."""
+Feel free to email us at [[email protected]](mailto:support@atla-ai.com) or leave feedback on our [Github](https://github.com/atla-ai/judge-arena)!"""

criteria_handler.py

@@ -0,0 +1,226 @@
+# criteria_handler.py
+
+import gradio as gr
+import re
+from eval_criteria_library import EXAMPLE_METRICS
+
+SYSTEM_PROMPT = """Please act as an impartial judge and evaluate based on the user's instruction. Your output format should strictly adhere to JSON as follows: {"feedback": "<write feedback>", "result": <numerical score>}. Ensure the output is valid JSON, without additional formatting or explanations."""
+
+EVALUATION_TEMPLATE = '''You are tasked with evaluating a response based on a given instruction (which may contain an Input) and a scoring rubric. Provide a comprehensive feedback on the response quality strictly adhering to the scoring rubric, without any general evaluation. Follow this with a score, referring to the scoring rubric. Avoid generating any additional opening, closing, or explanations.
+
+Here are some rules of the evaluation:
+(1) You should prioritize evaluating whether the response satisfies the provided rubric. The basis of your score should depend exactly on the rubric. However, the response does not need to explicitly address points raised in the rubric. Rather, evaluate the response based on the criteria outlined in the rubric.
+
+Your reply should strictly follow this format:
+Your output format should strictly adhere to JSON as follows: {% raw %}{"feedback": "<write feedback>", "result": <numerical score>}{% endraw %}. Ensure the output is valid JSON, without additional formatting or explanations.
+
+Here is the data.
+
+{% if model_context is defined and model_context %}Context:
+```
+{{ model_context }}
+```
+
+{% endif %}Instruction:
+```
+{{ model_input }}
+```
+
+Response:
+```
+{{ model_output }}
+```
+
+Score Rubrics:
+{{ evaluation_criteria }}
+
+{% if expected_model_output is defined and expected_model_output %}Reference answer:
+{{ expected_model_output }}{% endif %}'''
+
+def select_evaluation_criteria(data_upload_group, df_state, prompt_state):
+    with gr.Group(visible=True) as criteria_group:
+        select_eval_criteria_button = gr.Button("Select Evaluation Criteria", visible=False)
+        
+        criteria_dropdown = gr.Dropdown(
+            choices=list(EXAMPLE_METRICS.keys()),
+            label="Choose Evaluation Criteria",
+            value=list(EXAMPLE_METRICS.keys())[0],
+            visible=False
+        )
+
+        with gr.Row(visible=False) as mapping_row:
+            with gr.Column():
+                # Left column - Evaluation Criteria Editor
+                prompt_editor = gr.Textbox(
+                    label="Evaluation Criteria",
+                    lines=15,
+                    visible=False,
+                    placeholder="Enter the evaluation criteria/rubric here..."
+                )
+            with gr.Column():
+                # Right column - Required and Optional Variable Mapping
+                # Required mappings
+                input_mapping = gr.Dropdown(
+                    choices=[], 
+                    label="Map 'model_input' to column (Required)",
+                    interactive=True, 
+                    visible=False
+                )
+                output_mapping = gr.Dropdown(
+                    choices=[], 
+                    label="Map 'model_output' to column (Required)",
+                    interactive=True, 
+                    visible=False
+                )
+                # Optional mappings
+                context_mapping = gr.Dropdown(
+                    choices=[], 
+                    label="Map 'model_context' to column (Optional)",
+                    interactive=True, 
+                    visible=False
+                )
+                expected_output_mapping = gr.Dropdown(
+                    choices=[], 
+                    label="Map 'expected_model_output' to column (Optional)",
+                    interactive=True, 
+                    visible=False
+                )
+        # We'll place the "Back to Data" and "Select Evaluators" within the same row:
+        with gr.Row(visible=False) as nav_row:
+            back_to_data_button = gr.Button("← Back to Data", visible=False)
+            save_prompt_button = gr.Button("Select Evaluators", visible=False)
+
+        def update_column_choices(df_state):
+            df = df_state.value
+            columns = df.columns.tolist() if df is not None else []
+            return {
+                input_mapping: gr.update(choices=columns, visible=True),
+                output_mapping: gr.update(choices=columns, visible=True),
+                context_mapping: gr.update(choices=['None'] + columns, visible=True),
+                expected_output_mapping: gr.update(choices=['None'] + columns, visible=True)
+            }
+
+        def update_prompt(selected_criteria, df_state):
+            if selected_criteria in EXAMPLE_METRICS:
+                evaluation_criteria = EXAMPLE_METRICS[selected_criteria]['prompt']
+            else:
+                evaluation_criteria = ""
+            updates = {prompt_editor: gr.update(value=evaluation_criteria, visible=True)}
+            updates.update(update_column_choices(df_state))
+            return updates
+
+        def show_criteria_selection():
+            default_criterion = list(EXAMPLE_METRICS.keys())[0]
+            evaluation_criteria = EXAMPLE_METRICS[default_criterion]['prompt']
+            updates = {
+                select_eval_criteria_button: gr.update(visible=False),
+                criteria_dropdown: gr.update(visible=True),
+                prompt_editor: gr.update(value=evaluation_criteria, visible=True),
+                data_upload_group: gr.update(visible=False),
+                mapping_row: gr.update(visible=True),
+                # Show the nav row and buttons
+                nav_row: gr.update(visible=True),
+                back_to_data_button: gr.update(visible=True),
+                save_prompt_button: gr.update(visible=True),
+            }
+            updates.update(update_column_choices(df_state))
+            return updates
+
+        def save_prompt(evaluation_criteria, input_col, output_col, context_col, expected_output_col):
+            # Use the actual Jinja template with proper Jinja syntax and raw JSON
+            template = EVALUATION_TEMPLATE
+            
+            # Create mapping dictionary
+            mapping_dict = {
+                'model_input': input_col,
+                'model_output': output_col,
+                'evaluation_criteria': evaluation_criteria
+            }
+            
+            # Add optional mappings if selected
+            if context_col != 'None':
+                mapping_dict['model_context'] = context_col
+            if expected_output_col != 'None':
+                mapping_dict['expected_model_output'] = expected_output_col
+                
+            prompt_state.value = {
+                'template': template,
+                'mappings': mapping_dict
+            }
+
+        # Update event handlers
+        select_eval_criteria_button.click(
+            fn=show_criteria_selection,
+            inputs=[],
+            outputs=[
+                
+                select_eval_criteria_button,
+                criteria_dropdown,
+                prompt_editor,
+               
+                data_upload_group,
+                mapping_row,
+                nav_row,
+                back_to_data_button,
+                save_prompt_button
+            ,
+                input_mapping, output_mapping, context_mapping, expected_output_mapping
+            ]
+        )
+
+        criteria_dropdown.change(
+            fn=update_prompt,
+            inputs=[criteria_dropdown, df_state],
+            outputs=[prompt_editor, input_mapping, output_mapping, context_mapping, expected_output_mapping]
+        )
+
+        def make_select_button_visible(df_value):
+            if df_value is not None:
+                return gr.update(visible=True)
+            else:
+                return gr.update(visible=False)
+
+        df_state.change(
+            fn=make_select_button_visible,
+            inputs=df_state,
+            outputs=select_eval_criteria_button
+        )
+
+        save_prompt_button.click(
+            fn=save_prompt,
+            inputs=[
+                prompt_editor, input_mapping, output_mapping,
+                context_mapping, expected_output_mapping
+            ],
+            outputs=[]
+        )
+
+        # BACK BUTTON: Hide the criteria UI, show the data upload UI
+        def back_to_data():
+            return {
+                # show data upload group again
+                data_upload_group: gr.update(visible=True),
+                # hide the criteria group
+                criteria_dropdown: gr.update(visible=False),
+                prompt_editor: gr.update(visible=False),
+                mapping_row: gr.update(visible=False),
+                nav_row: gr.update(visible=False),
+                # make "Select Evaluation Criteria" button visible again
+                select_eval_criteria_button: gr.update(visible=True),
+            }
+
+        back_to_data_button.click(
+            fn=back_to_data,
+            inputs=[],
+            outputs=[
+                data_upload_group,
+                criteria_dropdown,
+                prompt_editor,
+                mapping_row,
+                nav_row,
+                select_eval_criteria_button
+            ]
+        )
+
+    # Return both the criteria rule group, the df_state, prompt_state, save_prompt_button
+    return criteria_group, df_state, prompt_state, save_prompt_button

data/models.jsonl

@@ -0,0 +1,17 @@
+{"name": "Meta Llama 3.1 70B Instruct Turbo", "organization": "Meta", "license": "Open Source", "api_model": "meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo"}
+{"name": "Meta Llama 3.1 405B Instruct Turbo", "organization": "Meta", "license": "Open Source", "api_model": "meta-llama/Meta-Llama-3.1-405B-Instruct-Turbo"}
+{"name": "Gemma 2 27B", "organization": "Google", "license": "Open Source", "api_model": "google/gemma-2-27b-it"}
+{"name": "Gemma 2 9B", "organization": "Google", "license": "Open Source", "api_model": "google/gemma-2-9b-it"}
+{"name": "Qwen 2 Instruct (72B)", "organization": "Alibaba", "license": "Open Source", "api_model": "Qwen/Qwen2-72B-Instruct"}
+{"name": "Mistral (7B) Instruct v0.3", "organization": "Mistral AI", "license": "Open Source", "api_model": "mistralai/Mistral-7B-Instruct-v0.3"}
+{"name": "GPT-4o", "organization": "OpenAI", "license": "Proprietary", "api_model": "gpt-4o"}
+{"name": "GPT-4 Turbo", "organization": "OpenAI", "license": "Proprietary", "api_model": "gpt-4-turbo"}
+{"name": "GPT-3.5 Turbo", "organization": "OpenAI", "license": "Proprietary", "api_model": "gpt-3.5-turbo"}
+{"name": "Claude 3 Haiku", "organization": "Anthropic", "license": "Proprietary", "api_model": "claude-3-haiku-20240307"}
+{"name": "Claude 3 Sonnet", "organization": "Anthropic", "license": "Proprietary", "api_model": "claude-3-sonnet-20240229"}
+{"name": "Claude 3 Opus", "organization": "Anthropic", "license": "Proprietary", "api_model": "claude-3-opus-20240229"}
+{"name": "Meta Llama 3.1 8B Instruct Turbo", "organization": "Meta", "license": "Open Source", "api_model": "meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo"}
+{"name": "Qwen 2.5 72B Instruct Turbo", "organization": "Alibaba", "license": "Open Source", "api_model": "Qwen/Qwen2.5-72B-Instruct-Turbo"}
+{"name": "Qwen 2.5 7B Instruct Turbo", "organization": "Alibaba", "license": "Open Source", "api_model": "Qwen/Qwen2.5-7B-Instruct-Turbo"}
+{"name": "Mistral (7B) Instruct v0.1", "organization": "Mistral AI", "license": "Open Source", "api_model": "mistralai/Mistral-7B-Instruct-v0.1"}
+{"name": "Atla Selene", "organization": "Atla", "license": "Proprietary", "api_model": "atla-selene"}

data_handler.py

@@ -0,0 +1,96 @@
+# data_handler.py
+
+import gradio as gr
+import pandas as pd
+import json
+def upload_test_data(df_state):
+    with gr.Group() as data_upload_group:
+        file_upload = gr.File(
+            label="Upload JSON with test data incl. true labels as integers or floats",
+            file_types=[".json"],
+        )
+        import_button = gr.Button("Import Data", visible=False)
+        # Show exactly 5 rows, no scrolling
+        df_display = gr.Dataframe(
+            visible=False,
+            elem_classes=["truncate_cells"],
+            label="Uploaded Data"
+        )
+        error_display = gr.Textbox(visible=False)
+
+    def display_file_info(file):
+        if file is not None:
+            return {
+                import_button: gr.update(visible=True),
+                error_display: gr.update(visible=False)  # Hide previous errors
+            }
+        else:
+            return {
+                import_button: gr.update(visible=False),
+                df_display: gr.update(visible=False),
+                error_display: gr.update(visible=False)  # Hide previous errors
+            }
+
+    def import_data(file):
+        if file is not None:
+            try:
+                loaded_json = json.load(open(file.name))
+
+                # Handle various common JSON structures
+                if isinstance(loaded_json, list):
+                    # Top-level list
+                    df = pd.json_normalize(loaded_json, sep=".")
+                elif isinstance(loaded_json, dict):
+                    # Dictionary could contain a "data" key or not
+                    if "data" in loaded_json and isinstance(loaded_json["data"], list):
+                        df = pd.json_normalize(loaded_json["data"], sep=".")
+                    else:
+                        # Flatten the top-level dictionary
+                        df = pd.json_normalize(loaded_json, sep=".")
+                else:
+                    raise ValueError("Unsupported JSON structure. Please provide a list or object.")
+
+                df_state.value = df
+
+                return {
+                    df_display: gr.update(value=df_state.value, visible=True),
+                    import_button: gr.update(visible=False),
+                    df_state: df_state,
+                    error_display: gr.update(visible=False)  # Hide previous errors
+                }
+            except json.JSONDecodeError:
+                return {
+                    df_display: gr.update(visible=False),
+                    error_display: gr.update(
+                        value="**Error:** Invalid JSON file. Please upload a valid JSON file.",
+                        visible=True
+                    ),
+                    import_button: gr.update(visible=True),
+                    df_state: None
+                }
+            except Exception as e:
+                return {
+                    df_display: gr.update(visible=False),
+                    error_display: gr.update(value=f"**Error:** {str(e)}", visible=True),
+                    import_button: gr.update(visible=True),
+                    df_state: None
+                }
+        else:
+            return {
+                df_display: gr.update(visible=False),
+                import_button: gr.update(visible=True),
+                df_state: None
+            }
+
+    file_upload.change(
+        fn=display_file_info,
+        inputs=file_upload,
+        outputs=[import_button, df_display, error_display]
+    )
+    import_button.click(
+        fn=import_data,
+        inputs=file_upload,
+        outputs=[df_display, import_button, df_state, error_display]
+    )
+
+    return data_upload_group, df_state

eval_criteria_library.py

@@ -0,0 +1,61 @@
+EXAMPLE_METRICS = {
+"Custom":  {
+    "prompt":
+    """Evaluate a chat bot response to a user's input based on [INSERT CRITERIA OR SELECT EXAMPLE FROM LIST]
+
+0: The response's answer does not meet all of the evaluation criteria.
+1: The response's answer meets all of the evaluation criteria.""",
+    },
+    "Relevance": {
+        "prompt": """Evaluate how well the response fulfill the requirements of the instruction by providing relevant information. This includes responding in accordance with the explicit and implicit purpose of given instruction.
+
+1: The response is completely unrelated to the instruction, or the model entirely misunderstands the instruction.
+2: Most of the key points in the response are irrelevant to the instruction, and the response misses major requirements of the instruction.
+3: Some major points in the response contain irrelevant information or miss some requirements of the instruction.
+4: The response is relevant to the instruction but misses minor requirements of the instruction.
+5: The response is perfectly relevant to the instruction, and the model fulfills all of the requirements of the instruction.""",
+    },
+    "Correctness": {
+        "prompt": """Evaluate whether the information provided in the response is correct given the reference response. Ignore differences in punctuation and phrasing between the student answer and true answer. It is okay if the student answer contains more information than the true answer, as long as it does not contain any conflicting statements.
+
+0: The response is not factually accurate when compared against the reference response or includes conflicting statements.
+1: The response is supported by the reference response and does not contain conflicting statements.""",
+    },
+    "Helpfulness": {
+        "prompt": """Evaluate how helpful the response is to address the user query.
+
+1: The response is not at all useful, failing to address the instruction or provide any valuable information.
+2: The response has minimal usefulness, addressing the instruction only superficially or providing mostly irrelevant information.
+3: The response is moderately useful, addressing some aspects of the instruction effectively but lacking in others.
+4: The response is very useful, effectively addressing most aspects of the instruction and providing valuable information.
+5: The response is exceptionally useful, fully addressing the instruction and providing highly valuable information.""",
+    },
+    "Faithfulness": {
+        "prompt": """Evaluate how well the statements in the response are directly supported by the context given in the related passages.
+
+1: The response contains statements that directly contradict the context or are entirely unsupported by it.
+2: The response includes some information from the context, but contains significant ungrounded claims or misinterpretations.
+3: The response is mostly grounded in the context, with only minor unsupported claims or misinterpretations.
+4: The response closely aligns with the context, with only rare and minor deviations.
+5: The response is fully grounded in the context, with all statements accurately reflecting the provided information.""",
+    },
+    "Logical coherence": {
+        "prompt": """Evaluate how logically accurate and correct the response is for the instruction given.
+
+1: The logic of the model’s response is completely incoherent.
+2: The model’s response contains major logical inconsistencies or errors.
+3: The model’s response contains some logical inconsistencies or errors, but they are not significant."
+4: The model’s response is logically sound, but it is slightly flawed in some aspect.
+5: The model’s response is logically flawless.""",
+    },
+    "Conciseness": {
+        "prompt": """Evaluate how concise the response is presented to the user without any unncecessary information.
+
+1: The response is highly redundant or contains a lot of unnecessary information, requiring a complete rewrite for optimal clarity and efficiency.
+2: The response lacks conciseness and needs a substantial rewrite for better optimization.
+3: The response is somewhat concise but includes unnecessary information, requiring
+some edits for improved optimization.
+4: The response is mostly concise but could benefit from minor edits for better optimization.
+5: The response is optimally concise and does not contain any unnecessary information, requiring no further optimization.""",
+    },
+}

get_llm_answer.py

@@ -0,0 +1,137 @@
+# get_llm_answer.py
+
+from openai import OpenAI
+import anthropic
+from together import Together
+import json
+import re
+from atla import Atla
+
+from dotenv import load_dotenv
+load_dotenv()
+
+# Initialize clients
+anthropic_client = anthropic.Anthropic()
+openai_client = OpenAI()
+together_client = Together()
+atla_client = Atla()
+
+SYSTEM_PROMPT = """Please act as an impartial judge and evaluate based on the user's instruction. Your output format should strictly adhere to JSON as follows: {"feedback": "<write feedback>", "result": <numerical score>}. Ensure the output is valid JSON, without additional formatting or explanations."""
+
+def get_openai_response(model_name, prompt):
+    """Get response from OpenAI API"""
+    try:
+        response = openai_client.chat.completions.create(
+            model=model_name,
+            messages=[
+                {"role": "system", "content": SYSTEM_PROMPT},
+                {"role": "user", "content": prompt},
+            ],
+        )
+        return response.choices[0].message.content
+    except Exception as e:
+        return f"Error with OpenAI model {model_name}: {str(e)}"
+
+
+def get_anthropic_response(model_name, prompt):
+    """Get response from Anthropic API"""
+    try:
+        response = anthropic_client.messages.create(
+            model=model_name,
+            max_tokens=1000,
+            temperature=0,
+            system=SYSTEM_PROMPT,
+            messages=[{"role": "user", "content": [{"type": "text", "text": prompt}]}],
+        )
+        return response.content[0].text
+    except Exception as e:
+        return f"Error with Anthropic model {model_name}: {str(e)}"
+
+
+def get_together_response(model_name, prompt):
+    """Get response from Together API"""
+    try:
+        response = together_client.chat.completions.create(
+            model=model_name,
+            messages=[
+                {"role": "system", "content": SYSTEM_PROMPT},
+                {"role": "user", "content": prompt},
+            ],
+            stream=False,
+        )
+        return response.choices[0].message.content
+    except Exception as e:
+        return f"Error with Together model {model_name}: {str(e)}"
+
+
+def get_atla_response(model_name, model_input, model_output, model_context, expected_output, evaluation_criteria):
+    """Get response from Atla API"""
+    try:
+        response = atla_client.evaluation.create(
+            model_id=model_name,
+            model_input=model_input,
+            model_output=model_output,
+            model_context=model_context,
+            expected_model_output=expected_output,
+            evaluation_criteria=evaluation_criteria,
+        )
+        # Return the score and critique directly from the evaluation result
+        return {
+            "score": response.result.evaluation.score,
+            "critique": response.result.evaluation.critique
+        }
+    except Exception as e:
+        return f"Error with Atla model {model_name}: {str(e)}"
+
+
+def get_model_response(model_name, model_info, prompt=None, **kwargs):
+    """Get response from appropriate API based on model organization"""
+    if not model_info:
+        return "Model not found or unsupported."
+
+    api_model = model_info["api_model"]
+    organization = model_info["organization"]
+
+    try:
+        if organization == "Atla":
+            return get_atla_response(
+                api_model,
+                kwargs.get('model_input'),
+                kwargs.get('model_output'),
+                kwargs.get('model_context'),
+                kwargs.get('expected_output'),
+                kwargs.get('evaluation_criteria')
+            )
+        elif organization == "OpenAI":
+            return get_openai_response(api_model, prompt)
+        elif organization == "Anthropic":
+            return get_anthropic_response(api_model, prompt)
+        else:
+            # All other organizations use Together API
+            return get_together_response(api_model, prompt)
+    except Exception as e:
+        return f"Error with {organization} model {model_name}: {str(e)}"
+
+
+def parse_model_response(response):
+    try:
+        # Debug print
+        print(f"Raw model response: {response}")
+
+        # First try to parse the entire response as JSON
+        try:
+            data = json.loads(response)
+            return str(data.get("result", "N/A")), data.get("feedback", "N/A")
+        except json.JSONDecodeError:
+            # If that fails (typically for smaller models), try to find JSON within the response
+            json_match = re.search(r"{.*}", response)
+            if json_match:
+                data = json.loads(json_match.group(0))
+                return str(data.get("result", "N/A")), data.get("feedback", "N/A")
+            else:
+                return "Error", f"Failed to parse response: {response}"
+
+    except Exception as e:
+        # Debug print for error case
+        print(f"Failed to parse response: {str(e)}")
+        return "Error", f"Failed to parse response: {response}"

model_handler.py

@@ -0,0 +1,221 @@
+# model_handler.py
+
+import gradio as gr
+import json
+import os
+import re
+from get_llm_answer import get_model_response, parse_model_response, get_atla_response
+from jinja2 import Template
+
+def select_evaluators(criteria_group, df_state, prompt_state, save_prompt_button):
+    with gr.Group(visible=True) as model_selection_group:
+        select_evaluators_button = gr.Button("Select Evaluators", visible=False)
+
+        # Load the model_data from JSONL
+        def load_model_data():
+            model_data = {}
+            try:
+                script_dir = os.path.dirname(__file__)
+                file_path = os.path.join(script_dir, "models.jsonl")
+                with open(file_path, "r") as f:
+                    for line in f:
+                        model = json.loads(line)
+                        model_data[model["name"]] = {
+                            "organization": model["organization"],
+                            "license": model["license"],
+                            "api_model": model["api_model"],
+                        }
+            except FileNotFoundError:
+                print("Warning: models.jsonl not found")
+                return {}
+            return model_data
+
+        model_data = load_model_data()
+        model_choices = list(model_data.keys())
+
+        with gr.Row(visible=False) as evaluator_row:
+               judge_a_dropdown = gr.Dropdown(
+                   choices=["Selene"], label="Judge A", value="Selene", interactive=False
+               )
+               judge_b_dropdown = gr.Dropdown(
+                   choices=model_choices, label="Judge B", value="Claude 3.5 Sonnet"
+               )
+
+        loading_spinner = gr.Markdown("Evaluation in progress...", visible=False)
+
+        evaluation_result_df = gr.Dataframe(
+            visible=False,
+            label="Evaluation Results",
+            elem_classes=["truncate_cells"]
+        )
+
+        with gr.Row(visible=False) as evaluation_nav_row:
+            back_to_criteria_button = gr.Button("← Back to Criteria", visible=False)
+            run_evaluation_button = gr.Button("Run Evaluation", visible=False)
+            analyze_results_button = gr.Button("Analyze Results", visible=False)
+
+        def show_evaluator_selection(current_df):
+            updates = {
+                criteria_group: gr.update(visible=False),
+                save_prompt_button: gr.update(visible=False),
+                evaluator_row: gr.update(visible=True),
+                evaluation_nav_row: gr.update(visible=True),
+                run_evaluation_button: gr.update(visible=True),
+                back_to_criteria_button: gr.update(visible=True),
+                analyze_results_button: gr.update(visible=False),
+                evaluation_result_df: gr.update(visible=False),
+            }
+            if (
+                current_df.value is not None
+                and hasattr(current_df.value, "attrs")
+                and current_df.value.attrs.get("eval_done")
+            ):
+                updates[loading_spinner] = gr.update(value="### Evaluation Complete", visible=True)
+                updates[evaluation_result_df] = gr.update(value=current_df.value, visible=True)
+                updates[analyze_results_button] = gr.update(visible=True)
+
+            return updates
+
+        save_prompt_button.click(
+            fn=show_evaluator_selection,
+            inputs=[df_state],
+            outputs=[
+                save_prompt_button,
+                criteria_group,
+                evaluator_row,
+                evaluation_nav_row,
+                run_evaluation_button,
+                back_to_criteria_button,
+                loading_spinner,
+                analyze_results_button,
+                evaluation_result_df,
+            ],
+        )
+
+        def back_to_criteria():
+            return {
+                save_prompt_button: gr.update(visible=True),
+                criteria_group: gr.update(visible=True),
+                evaluator_row: gr.update(visible=False),
+                evaluation_nav_row: gr.update(visible=False),
+                run_evaluation_button: gr.update(visible=False),
+                loading_spinner: gr.update(visible=False),
+                analyze_results_button: gr.update(visible=False),
+                evaluation_result_df: gr.update(visible=False),
+            }
+
+        back_to_criteria_button.click(
+            fn=back_to_criteria,
+            inputs=[],
+            outputs=[
+                save_prompt_button,
+                criteria_group,
+                evaluator_row,
+                evaluation_nav_row,
+                run_evaluation_button,
+                loading_spinner,
+                analyze_results_button,
+                evaluation_result_df
+            ],
+        )
+
+        # Run evaluation
+        def run_evaluation(judge_a, judge_b):
+            # 1) Immediately hide old results and disable navigation while running
+            yield {
+                loading_spinner: gr.update(value="Evaluation in progress...", visible=True),
+                evaluation_result_df: gr.update(visible=False),
+                analyze_results_button: gr.update(visible=False),
+                run_evaluation_button: gr.update(interactive=False),
+                back_to_criteria_button: gr.update(interactive=False),
+            }
+
+            # Perform the actual evaluation
+            template_str = prompt_state.value['template']
+            mappings = prompt_state.value['mappings']
+            evaluation_criteria = mappings.get('evaluation_criteria')
+
+            template = Template(template_str)
+
+            for index, row in df_state.value.iterrows():
+                context = {}
+                model_context = None
+                expected_output = None
+
+                for key, column in mappings.items():
+                    if key == 'evaluation_criteria':
+                        continue
+                    elif column and column != 'None':
+                        context[key] = str(row[column])
+                        if column == 'model_context':
+                            model_context = str(row[column])
+                        elif column == 'expected_model_output':
+                            expected_output = str(row[column])
+
+                context['evaluation_criteria'] = evaluation_criteria
+
+                # Render the template for Judge B
+                current_prompt = template.render(**context)
+                print(f"\nDEBUG - Final Prompt sent to Model B:\n{current_prompt}\n")
+                response_a = get_atla_response(
+                    "atla-selene",
+                    model_input=context.get('model_input'),
+                    model_output=context.get('model_output'),
+                    model_context=model_context,
+                    expected_output=expected_output,
+                    evaluation_criteria=evaluation_criteria
+                )
+                response_b = get_model_response(
+                    judge_b,
+                    model_data.get(judge_b),
+                    current_prompt
+                )
+
+                # Parse ATLA response
+                if isinstance(response_a, dict):
+                    score_a, critique_a = response_a['score'], response_a['critique']
+                else:
+                    score_a, critique_a = "Error", response_a
+
+                score_b, critique_b = parse_model_response(response_b)
+
+                # Convert model name to snake case for column names
+                model_b_snake = judge_b.lower().replace(' ', '_').replace('-', '_').replace('.', '_')
+
+                df_state.value.loc[index, 'score_selene'] = score_a
+                df_state.value.loc[index, 'critique_selene'] = critique_a
+                df_state.value.loc[index, f'score_{model_b_snake}'] = score_b
+                df_state.value.loc[index, f'critique_{model_b_snake}'] = critique_b
+
+            import time
+            time.sleep(2)  # simulating time-consuming operations
+
+            # 2) Hide spinner
+            yield {loading_spinner: gr.update(visible=False)}
+
+            # 3) Show final results and re-enable buttons
+            yield {
+                loading_spinner: gr.update(value="### Evaluation Complete", visible=True),
+                evaluation_result_df: gr.update(value=df_state.value, visible=True),
+                analyze_results_button: gr.update(visible=True),
+                run_evaluation_button: gr.update(interactive=True),
+                back_to_criteria_button: gr.update(interactive=True),
+            }
+
+            if hasattr(df_state.value, "attrs"):
+                df_state.value.attrs["eval_done"] = True
+
+        # Include back_to_criteria_button & run_evaluation_button in outputs so we can update them
+        run_evaluation_button.click(
+            fn=run_evaluation,
+            inputs=[judge_a_dropdown, judge_b_dropdown],
+            outputs=[
+                loading_spinner,
+                evaluation_result_df,
+                analyze_results_button,
+                run_evaluation_button,
+                back_to_criteria_button,
+            ],
+        )
+
+    return model_selection_group, df_state, analyze_results_button

models.jsonl

@@ -0,0 +1,19 @@
+{"name": "Meta Llama 3.1 70B Instruct", "organization": "Meta", "license": "Open Source", "api_model": "meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo"}
+{"name": "Meta Llama 3.1 405B Instruct", "organization": "Meta", "license": "Open Source", "api_model": "meta-llama/Meta-Llama-3.1-405B-Instruct-Turbo"}
+{"name": "Gemma 2 27B", "organization": "Google", "license": "Open Source", "api_model": "google/gemma-2-27b-it"}
+{"name": "Gemma 2 9B", "organization": "Google", "license": "Open Source", "api_model": "google/gemma-2-9b-it"}
+{"name": "Qwen 2 Instruct (72B)", "organization": "Alibaba", "license": "Open Source", "api_model": "Qwen/Qwen2-72B-Instruct"}
+{"name": "Mistral (7B) Instruct v0.3", "organization": "Mistral AI", "license": "Open Source", "api_model": "mistralai/Mistral-7B-Instruct-v0.3"}
+{"name": "GPT-4o", "organization": "OpenAI", "license": "Proprietary", "api_model": "gpt-4o"}
+{"name": "GPT-4 Turbo", "organization": "OpenAI", "license": "Proprietary", "api_model": "gpt-4-turbo"}
+{"name": "GPT-3.5 Turbo", "organization": "OpenAI", "license": "Proprietary", "api_model": "gpt-3.5-turbo"}
+{"name": "Claude 3 Haiku", "organization": "Anthropic", "license": "Proprietary", "api_model": "claude-3-haiku-20240307"}
+{"name": "Claude 3 Sonnet", "organization": "Anthropic", "license": "Proprietary", "api_model": "claude-3-sonnet-20240229"}
+{"name": "Claude 3 Opus", "organization": "Anthropic", "license": "Proprietary", "api_model": "claude-3-opus-latest"}
+{"name": "Meta Llama 3.1 8B Instruct", "organization": "Meta", "license": "Open Source", "api_model": "meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo"}
+{"name": "Qwen 2.5 72B Instruct", "organization": "Alibaba", "license": "Open Source", "api_model": "Qwen/Qwen2.5-72B-Instruct-Turbo"}
+{"name": "Qwen 2.5 7B Instruct", "organization": "Alibaba", "license": "Open Source", "api_model": "Qwen/Qwen2.5-7B-Instruct-Turbo"}
+{"name": "Mistral (7B) Instruct v0.1", "organization": "Mistral AI", "license": "Open Source", "api_model": "mistralai/Mistral-7B-Instruct-v0.1"}
+{"name": "Claude 3.5 Sonnet", "organization": "Anthropic", "license": "Proprietary", "api_model": "claude-3-5-sonnet-latest"}
+{"name": "Claude 3.5 Haiku", "organization": "Anthropic", "license": "Proprietary", "api_model": "claude-3-5-haiku-latest"}
+{"name": "Atla Selene", "organization": "Atla", "license": "Proprietary", "api_model": "atla-selene"}

random_sample/__init__.py

@@ -0,0 +1 @@
+# This file can be empty - it just marks the directory as a Python package 

random_sample/arena_interface.py

@@ -0,0 +1,405 @@
+import json
+import re
+import gradio as gr
+
+from dotenv import load_dotenv
+load_dotenv()
+
+from .gen_api_answer import (
+    get_atla_response,
+    get_selene_mini_response,
+    parse_selene_mini_response
+)
+
+from .prompts import (
+    DEFAULT_EVAL_CRITERIA,
+    DEFAULT_EVAL_PROMPT,
+    DEFAULT_EVAL_PROMPT_EDITABLE,
+    ATLA_PROMPT,
+    ATLA_PROMPT_WITH_REFERENCE
+)
+
+from .random_sample_generation import (
+    get_random_human_ai_pair,
+    get_random_human_ai_ground_truth_pair,
+    generate_ai_response
+)   
+
+from common import CSS_STYLES, MAIN_TITLE, HOW_IT_WORKS
+
+def parse_variables(prompt):
+    # Extract variables enclosed in double curly braces
+    variables = re.findall(r"{{(.*?)}}", prompt)
+    # Remove duplicates while preserving order
+    seen = set()
+    variables = [
+        x.strip() for x in variables if not (x.strip() in seen or seen.add(x.strip()))
+    ]
+    return variables
+
+
+def get_final_prompt(eval_prompt, variable_values):
+    # Replace variables in the eval prompt with their values
+    for var, val in variable_values.items():
+        eval_prompt = eval_prompt.replace("{{" + var + "}}", val)
+    return eval_prompt
+
+
+def populate_random_example(request: gr.Request, compatible_mode: bool):
+    """Generate a random human-AI conversation example and reset judge outputs."""
+    if compatible_mode:
+        human_msg, ai_msg, ground_truth_msg = get_random_human_ai_ground_truth_pair()
+    else:
+        human_msg, ai_msg = get_random_human_ai_pair()
+        ground_truth_msg = ""
+    
+    return [
+        gr.update(value=human_msg),
+        gr.update(value=ai_msg),
+        gr.update(value="🎲", variant="secondary"),
+        gr.update(value=""),  # Clear score
+        gr.update(value=""),  # Clear critique
+        gr.update(value=ground_truth_msg, visible=compatible_mode),  # Set ground truth and visibility
+    ]
+
+
+def create_arena_interface():
+    with gr.Blocks(theme="default", css=CSS_STYLES) as interface:
+        # Hidden eval prompt that will always contain DEFAULT_EVAL_PROMPT
+        eval_prompt = gr.Textbox(
+            value=DEFAULT_EVAL_PROMPT,
+            visible=False
+        )
+        with gr.Row():
+            # Add model selector dropdown at the top
+            model_selector = gr.Dropdown(
+                choices=["Selene", "Selene Mini"],
+                value="Selene",
+                label="Choose your Atla Model",
+                interactive=True
+            )
+
+        with gr.Row():
+            # Left side - Input section
+            with gr.Column(scale=1):
+                with gr.Group():
+                    human_input = gr.TextArea(
+                        label="👩 User Input",
+                        lines=5,
+                        placeholder="Enter the human message here..."
+                    )
+                    with gr.Row():
+                        generate_btn = gr.Button(
+                            "Generate AI Response",
+                            size="sm",
+                            interactive=False
+                        )
+                    
+                    ai_response = gr.TextArea(
+                        label="🤖 AI Response", 
+                        lines=10,
+                        placeholder="Enter the AI response here..."
+                    )
+                    
+                    # Ground truth response (initially hidden)
+                    ground_truth = gr.TextArea(
+                        label="🎯 Ground truth response",
+                        lines=10,
+                        placeholder="Enter the ground truth response here...",
+                        visible=False
+                    )
+                    
+                with gr.Row():
+                    random_btn = gr.Button("🎲", scale=2)
+                    send_btn = gr.Button(
+                        value="Run evaluation",
+                        variant="primary",
+                        size="lg",
+                        scale=8
+                    )
+
+            # Right side - Model outputs
+            with gr.Column(scale=1):
+                gr.Markdown("## 👩‍⚖️ Selene-Mini Evaluation")
+                with gr.Group():
+                    with gr.Row():
+                        score = gr.Textbox(label="Score", lines=1, interactive=False)
+                    critique = gr.TextArea(label="Critique", lines=12, interactive=False)
+        
+        gr.Markdown("<br>")
+        
+
+        # Replace the "Edit Judge Prompt" Accordion section with:
+        with gr.Accordion("📝 Edit Judge Prompt", open=False) as prompt_accordion:
+            gr.Markdown("<br>")
+            use_reference_toggle = gr.Checkbox(
+                label="Use a reference response",
+                value=False
+            )
+            
+            # Hide the default prompt editor
+            with gr.Column(visible=False) as default_prompt_editor:
+                eval_prompt_editable = gr.TextArea(
+                    value=DEFAULT_EVAL_PROMPT_EDITABLE,
+                    label="Evaluation Criteria",
+                    lines=12
+                )
+
+                with gr.Row(visible=False) as edit_buttons_row:
+                    cancel_prompt_btn = gr.Button("Cancel")
+                    save_prompt_btn = gr.Button("Save", variant="primary")
+            
+            # Show the compatible mode editor
+            with gr.Column(visible=True) as compatible_prompt_editor:
+                eval_criteria_text = gr.TextArea(
+                    label="Evaluation Criteria",
+                    lines=12,
+                    value=DEFAULT_EVAL_CRITERIA,
+                    placeholder="Enter the complete evaluation criteria and scoring rubric..."
+                )
+                with gr.Row(visible=False) as compatible_edit_buttons_row:
+                    compatible_cancel_btn = gr.Button("Cancel")
+                    compatible_save_btn = gr.Button("Save", variant="primary")
+
+        eval_prompt_previous = gr.State(value=DEFAULT_EVAL_PROMPT_EDITABLE)  # Initialize with default value
+        is_editing = gr.State(False)  # Track editing state
+        compatible_mode_state = gr.State(False)  # Track compatible mode state
+
+        # Update model names after responses are generated
+        def update_model_names(model_a, model_b):
+            return gr.update(value=f"*Model: {model_a}*"), gr.update(
+                value=f"*Model: {model_b}*"
+            )
+
+        # Store the last submitted prompt and variables for comparison
+        last_submission = gr.State({})
+
+        # Update the save/cancel buttons section in the compatible prompt editor
+        def save_criteria(new_criteria, previous_criteria):
+            return [
+                gr.update(value=new_criteria),  # Update the criteria
+                new_criteria,  # Update the previous criteria state
+                gr.update(visible=False)  # Hide the buttons
+            ]
+
+        def cancel_criteria(previous_criteria):
+            return [
+                gr.update(value=previous_criteria),  # Revert to previous criteria
+                previous_criteria,  # Keep the previous criteria state
+                gr.update(visible=False)  # Hide the buttons
+            ]
+
+        def show_criteria_edit_buttons(current_value, previous_value):
+            # Show buttons only if the current value differs from the previous value
+            return gr.update(visible=current_value != previous_value)
+
+        # Add handlers for save/cancel buttons and criteria changes
+        compatible_save_btn.click(
+            fn=save_criteria,
+            inputs=[eval_criteria_text, eval_prompt_previous],
+            outputs=[eval_criteria_text, eval_prompt_previous, compatible_edit_buttons_row]
+        )
+
+        compatible_cancel_btn.click(
+            fn=cancel_criteria,
+            inputs=[eval_prompt_previous],
+            outputs=[eval_criteria_text, eval_prompt_previous, compatible_edit_buttons_row]
+        )
+
+        eval_criteria_text.change(
+            fn=show_criteria_edit_buttons,
+            inputs=[eval_criteria_text, eval_prompt_previous],
+            outputs=compatible_edit_buttons_row
+        )
+
+        # Function to toggle visibility based on compatible mode
+        def toggle_use_reference(checked):
+            if checked:
+                human_msg, ai_msg, ground_truth_msg = get_random_human_ai_ground_truth_pair()
+                return {
+                    ground_truth: gr.update(visible=True, value=ground_truth_msg),
+                    human_input: gr.update(value=human_msg),
+                    ai_response: gr.update(value=ai_msg),
+                    score: gr.update(value=""),
+                    critique: gr.update(value=""),
+                    random_btn: gr.update(value="🎲", variant="secondary"),
+                }
+            else:
+                return {
+                    ground_truth: gr.update(visible=False)
+                }
+
+        # Update the change handler to include all necessary outputs
+        use_reference_toggle.change(
+            fn=toggle_use_reference,
+            inputs=[use_reference_toggle],
+            outputs=[
+                ground_truth,
+                human_input,
+                ai_response,
+                score,
+                critique,
+                random_btn,
+            ]
+        )
+
+        # Add a new state variable to track first game
+        first_game_state = gr.State(True)  # Initialize as True
+
+        # Update the submit function to handle both models
+        def submit_and_store(
+            model_choice,
+            use_reference,
+            eval_criteria_text,
+            human_input,
+            ai_response,
+            ground_truth,
+        ):
+            if model_choice == "Selene Mini":
+                # Prepare prompt based on reference mode
+                prompt_template = ATLA_PROMPT_WITH_REFERENCE if use_reference else ATLA_PROMPT
+                prompt = prompt_template.format(
+                    human_input=human_input,
+                    ai_response=ai_response,
+                    eval_criteria=eval_criteria_text,
+                    ground_truth=ground_truth if use_reference else ""
+                )
+                
+                print("\n=== Debug: Prompt being sent to Selene Mini ===")
+                print(prompt)
+                print("============================================\n")
+                
+                # Get and parse response
+                raw_response = get_selene_mini_response(
+                    model_name="AtlaAI/Selene-1-Mini-Llama-3.1-8B",
+                    prompt=prompt,
+                    max_tokens=500,
+                    temperature=0.01
+                )
+                response = parse_selene_mini_response(raw_response)
+            else:
+                # Selene API logic
+                prompt_data = {
+                    'human_input': human_input,
+                    'ai_response': ai_response,
+                    'ground_truth': ground_truth if use_reference else None,
+                    'eval_criteria': eval_criteria_text,
+                }
+                
+                print("\n=== Debug: Prompt data being sent to Selene API ===")
+                print(json.dumps(prompt_data, indent=2))
+                print("============================================\n")
+                
+                response = get_atla_response(
+                    model_name="AtlaAI/Selene-1-Mini-Llama-3.1-8B",
+                    prompt=prompt_data,
+                    max_tokens=500,
+                    temperature=0.01
+                )
+
+            # Response now contains score and critique directly
+            if isinstance(response, dict) and 'score' in response and 'critique' in response:
+                score = str(response['score'])
+                critique = response['critique']
+            else:
+                score = "Error"
+                critique = str(response)
+
+            return [
+                score,
+                critique,
+                gr.update(value="Regenerate evaluation", variant="secondary", interactive=True),
+                gr.update(value="🎲"),
+            ]
+
+        # Update the send_btn click handler with new input
+        send_btn.click(
+            fn=submit_and_store,
+            inputs=[
+                model_selector,
+                use_reference_toggle,
+                eval_criteria_text,
+                human_input,
+                ai_response,
+                ground_truth,
+            ],
+            outputs=[
+                score,
+                critique,
+                send_btn,
+                random_btn,
+            ],
+        )
+
+        # Add random button handler
+        random_btn.click(
+            fn=populate_random_example,
+            inputs=[use_reference_toggle],
+            outputs=[
+                human_input,
+                ai_response,
+                random_btn,
+                score,
+                critique,
+                ground_truth,
+            ]
+        )
+
+        # Add input change handlers
+        def handle_input_change():
+            """Reset UI state when inputs are changed"""
+            return [
+                gr.update(value="Run evaluation", variant="primary"),  # send_btn
+                gr.update(value="🎲", variant="secondary"),  # random_btn
+            ]
+
+        # Update the change handlers for inputs
+        human_input.change(
+            fn=handle_input_change,
+            inputs=[],
+            outputs=[send_btn, random_btn]
+        )
+
+        ai_response.change(
+            fn=handle_input_change,
+            inputs=[],
+            outputs=[send_btn, random_btn]
+        )
+
+        generate_btn.click(
+            fn=lambda msg: (
+                generate_ai_response(msg)[0],  # Only take the response text
+                gr.update(
+                    value="Generate AI Response",  # Keep the label
+                    interactive=False  # Disable the button
+                )
+            ),
+            inputs=[human_input],
+            outputs=[ai_response, generate_btn]
+        )
+
+        human_input.change(
+            fn=lambda x: gr.update(interactive=bool(x.strip())),
+            inputs=[human_input],
+            outputs=[generate_btn]
+        )
+
+        # Update the demo.load to include the random example population
+        interface.load(
+            fn=lambda: populate_random_example(None, False),  # Pass False for initial compatible_mode
+            inputs=[],
+            outputs=[
+                human_input,
+                ai_response,
+                random_btn,
+                score,
+                critique,
+                ground_truth,
+            ]
+        )
+
+    return interface
+
+if __name__ == "__main__":
+    demo = create_arena_interface()
+    demo.launch()

random_sample/common.py

@@ -0,0 +1,126 @@
+# Page Headers
+MAIN_TITLE = "# Selene-Mini"
+
+# How it works section
+HOW_IT_WORKS = """
+Try running evals with Selene-Mini in this playground! Our HF model card can be found [here](https://huggingface.co/AtlaAI/Selene-1-Mini-Llama-3.1-8B).
+"""
+
+BATTLE_RULES = """
+## 🤺 Choose the winner
+1. Define your scoring criteria in the **Evaluator Prompt** 
+2. Add a test case to the **Sample to evaluate**
+3. Test the evaluators & vote for the model that best aligns with your judgement!
+\n
+Variables defined in your prompt with {{double curly braces}} map to input fields under **Sample to evaluate**.
+
+<br>
+"""
+
+# CSS Styles
+CSS_STYLES = """
+    .prompt-row {
+        align-items: flex-start !important;
+    }
+    .send-button-row {
+        display: flex;
+        justify-content: flex-end;
+        margin-top: 8px;
+    }
+    /* Style for metric buttons */
+    .metric-button-active {
+        background-color: #2B3A55 !important;
+        color: white !important;
+    }
+    /* Add this to ensure proper button spacing */
+    .metric-buttons-row {
+        gap: 8px;
+    }
+"""
+
+# Default Eval Prompt
+EVAL_DESCRIPTION = """
+## 📝 Tips
+**Precise evaluation criteria leads to more consistent and reliable judgments.** A good evaluation prompt should include the following elements:
+- Evaluation criteria
+- Scoring rubric 
+- Examples (Optional)
+"""
+
+# Voting Section Header
+VOTING_HEADER = """
+# Start Voting Now
+"""
+
+# Acknowledgements
+ACKNOWLEDGEMENTS = """
+<br><br>
+# Acknowledgements
+
+We thank [LMSYS Org](https://lmsys.org/) for their hard work on the Chatbot Arena and fully credit them for the inspiration to build this.
+
+We thank [Clementine Fourrier](https://huggingface.co/clefourrier) and Hugging Face for their guidance and partnership in setting this up.
+"""
+
+# Policy Content
+POLICY_CONTENT = """
+# About Atla
+
+Atla is an applied research organization that trains models as evaluators to capture human preferences. We're a team of researchers, engineers, and operational leaders, with experience spanning a variety of disciplines, all working together to build reliable and understandable AI systems. Our research is informed by our experiences conducting AI safety research at the UK AI Task Force, OpenAI and the Stanford Existential Risks Initiative.
+<br><br>
+# [Our Mission](https://www.atla-ai.com/company)
+
+By creating advanced evaluation models, we enable AI developers to identify and fix risks, leading to safer, more reliable AI that can be trusted and widely used. Our aim is to surpass the current state-of-the-art evaluation methods by training models specifically for evaluation. AIs will probably become very powerful, and perform tasks that are difficult for us to verify. We want to enable humans to oversee AI systems that are solving tasks too difficult for humans to evaluate. 
+Read more about [our approach to scalable oversight](https://www.atla-ai.com/post/scaling-alignment) on our blog.
+<br><br>
+# Judge Arena Policy
+
+## Overview
+
+Judge Arena is an open-source platform dedicated to determining which models make the best judges. Users can run evals and assess anonymized responses from two competing model judges, choosing the better judgement or declaring a tie. This policy outlines our commitments to maintain a fair and open environment :)
+
+## Transparency
+
+- **Open-Source**: Judge Arena's code is open-source and available on GitHub. We encourage contributions from the community and anyone can replicate or modify the platform to suit their needs. We use proprietary model provider APIs where provided and Together AI's API to serve leading open-source models.
+- **Methodology**: All processes related to model evaluation, rating calculations, and model selection are openly documented. 
+- **Data Sharing**: Periodically, we'll share 20% of the collected evaluation data with the community. The data collected from Judge Arena is restricted to an anonymized user ID, the final prompt sent, the model responses, the user vote, and the timestamp.
+
+## Model Inclusion Criteria
+
+Judge Arena is specifically designed to assess AI models that function as evaluators (a.k.a judges). This includes but is not limited to powerful general-purpose models and the latest language models designed for evaluation tasks. Models are eligible for inclusion if they meet the following criteria:
+
+- **Judge Capability**: The model should possess the ability to score AND critique other models' outputs effectively.
+- **Promptable:** The model must be promptable to be evaluate in different scoring formats, for different criteria.
+- **Accessibility**:
+   - **Public API Access**: Models accessible through public APIs without restrictive barriers.
+   - **Open-Source Models**: Models with publicly available weights that can be downloaded and run by the community.
+
+## Leaderboard Management
+
+- **ELO Ranking System**: Models are ranked on a public leaderboard based on aggregated user evaluations. We use an ELO rating system to rank AI judges on the public leaderboard. Each model begins with an initial rating of 1200, and we use a K-factor of 32 to determine the maximum rating adjustment after each evaluation.
+- **Minimum Period**: Listed models remain accessible on Judge Arena for a minimum period of two weeks so they can be comprehensively evaluated.
+- **Deprecation Policy**: Models may be removed from the leaderboard if they become inaccessible or are no longer publicly available.
+
+*This policy might be updated to reflect changes in our practices or in response to community feedback.*
+<br><br>
+# FAQ
+
+**Isn't this the same as Chatbot Arena?**
+
+We are big fans of what the LMSYS team have done with Chatbot Arena and fully credit them for the inspiration to develop this. We were looking for a dynamic leaderboard that graded on AI judge capabilities and didn't manage to find one, so we created Judge Arena. This UI is designed especially for evals; to match the format of the model-based eval prompts that you would use in your LLM evaluation / monitoring tool.
+
+**Why should I trust this leaderboard?**
+
+We have listed out our efforts to be fully transparent in the policies above. All of the code for this leaderboard is open-source and can be found on our [Github](https://github.com/atla-ai/judge-arena). Check out our [blog](https://www.atla-ai.com/blog) to stay up to date as we analyse the results from the leaderboard.
+
+**Who funds this effort?**
+
+Atla currently funds this out of our own pocket. We are looking for API credits (with no strings attached) to support this effort - please get in touch if you or someone you know might be able to help.
+
+**What is Atla working on?**
+
+We are training a general-purpose evaluator that you will soon be able to run in this Judge Arena. Our next step will be to open-source a powerful model that the community can use to run fast and accurate evaluations.
+<br><br>
+# Get in touch
+We’d love to hear your feedback! For general feature requests or to submit / suggest new models to add to the arena, please open up a discussion in the [community](https://huggingface.co/spaces/AtlaAI/judge-arena/discussions) tab. You can also contact us directly on [X](https://x.com/Atla_AI) or [Discord](https://discord.gg/yNpUAMqs).
+\nPlease file any issues on our [Github](https://github.com/atla-ai/judge-arena)."""

random_sample/gen_api_answer.py

@@ -0,0 +1,148 @@
+from openai import OpenAI
+import anthropic
+from together import Together
+import os       
+from atla import Atla
+from dotenv import load_dotenv
+from .prompts import (
+    JUDGE_SYSTEM_PROMPT
+)
+from transformers import AutoTokenizer
+import requests
+import json
+import re
+
+load_dotenv()
+
+# Initialize clients
+anthropic_client = anthropic.Anthropic()
+openai_client = OpenAI()
+together_client = Together()
+hf_api_key = os.getenv("HF_API_KEY")
+
+atla_client = Atla()
+
+def get_openai_response(model_name, prompt, system_prompt=JUDGE_SYSTEM_PROMPT, max_tokens=500, temperature=0):
+    """Get response from OpenAI API"""
+    try:
+        response = openai_client.chat.completions.create(
+            model=model_name,
+            messages=[
+                {"role": "system", "content": system_prompt},
+                {"role": "user", "content": prompt},
+            ],
+            max_completion_tokens=max_tokens,
+            temperature=temperature,
+        )
+        return response.choices[0].message.content
+    except Exception as e:
+        return f"Error with OpenAI model {model_name}: {str(e)}"
+
+def get_anthropic_response(model_name, prompt, system_prompt=JUDGE_SYSTEM_PROMPT, max_tokens=500, temperature=0):
+    """Get response from Anthropic API"""
+    try:
+        response = anthropic_client.messages.create(
+            model=model_name,
+            max_tokens=max_tokens,
+            temperature=temperature,
+            system=system_prompt,
+            messages=[{"role": "user", "content": [{"type": "text", "text": prompt}]}],
+        )
+        return response.content[0].text
+    except Exception as e:
+        return f"Error with Anthropic model {model_name}: {str(e)}"
+
+
+def get_atla_response(model_name, prompt, system_prompt=None, max_tokens=500, temperature=0.01):
+    """Get response from Atla API"""
+    try:
+        # Extract components from the prompt data
+        model_input = prompt.get('human_input', '')
+        model_output = prompt.get('ai_response', '')
+        expected_output = prompt.get('ground_truth')
+        evaluation_criteria = prompt.get('eval_criteria', '')
+
+        response = atla_client.evaluation.create(
+            model_id="atla-selene",
+            model_input=model_input,
+            model_output=model_output,
+            expected_model_output=expected_output if expected_output else None,
+            evaluation_criteria=evaluation_criteria,
+        )
+        
+        # Return the score and critique directly
+        return {
+            "score": response.result.evaluation.score,
+            "critique": response.result.evaluation.critique
+        }
+    except Exception as e:
+        return f"Error with Atla model {model_name}: {str(e)}"
+
+def get_selene_mini_response(model_name, prompt, system_prompt=None, max_tokens=500, temperature=0.01):
+    """Get response from HF endpoint for Atla model"""
+    try:
+        headers = {
+            "Accept": "application/json",
+            "Authorization": f"Bearer {hf_api_key}",
+            "Content-Type": "application/json"
+        }
+        
+        # Create messages list for chat template
+        messages = []
+        if system_prompt:
+            messages.append({"role": "system", "content": system_prompt})
+        messages.append({"role": "user", "content": prompt})
+        
+        # Apply chat template
+        model_id = "AtlaAI/Selene-1-Mini-Llama-3.1-8B"
+        tokenizer = AutoTokenizer.from_pretrained(model_id, token=hf_api_key)
+        formatted_prompt = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
+        
+        payload = {
+            "inputs": formatted_prompt,
+            "parameters": {
+                "max_new_tokens": max_tokens,
+                "return_full_text": False,
+                "temperature": temperature,
+                "seed": 42,
+                "add_generation_prompt": True
+            }
+        }
+        
+        response = requests.post(
+            "https://bkp9p28gri93egqh.us-east-1.aws.endpoints.huggingface.cloud",
+            headers=headers,
+            json=payload
+        )
+        return response.json()[0]["generated_text"]
+    except Exception as e:
+        return f"Error with Atla model {model_name}: {str(e)}"
+
+def parse_selene_mini_response(response_text):
+    """Parse the response from Selene Mini to extract score and critique"""
+    try:
+        # Clean up the response text
+        response_text = response_text.strip()
+        
+        # More flexible regex patterns
+        reasoning_pattern = r'\*\*Reasoning:?\*\*\s*(.*?)(?=\*\*Result|$)'
+        result_pattern = r'\*\*Result:?\*\*\s*(\d+)'
+        
+        reasoning_match = re.search(reasoning_pattern, response_text, re.DOTALL | re.IGNORECASE)
+        result_match = re.search(result_pattern, response_text, re.IGNORECASE)
+        
+        if reasoning_match and result_match:
+            critique = reasoning_match.group(1).strip()
+            score = result_match.group(1)
+            return {"score": score, "critique": critique}
+        else:
+            # If we can't parse it properly, let's return the raw response as critique
+            return {
+                "score": "Error",
+                "critique": f"Failed to parse response. Raw response:\n{response_text}"
+            }
+    except Exception as e:
+        return {
+            "score": "Error",
+            "critique": f"Error parsing response: {str(e)}\nRaw response:\n{response_text}"
+        }

random_sample/prompts.py

@@ -0,0 +1,94 @@
+# Default values for compatible mode
+DEFAULT_EVAL_CRITERIA = """Does the model provide relevant and useful responses to the user's needs or questions?
+
+Scoring Rubric:
+Score 1: The model's responses are irrelevant or unhelpful to the user's needs or queries.
+Score 2: The model sometimes provides helpful information, but often fails to address the user's actual needs or questions.
+Score 3: The model generally provides helpful responses that address the user's needs, though it may occasionally miss the mark.
+Score 4: The model regularly provides helpful responses that are well-aligned with the user's inquiries, with only rare inaccuracies.
+Score 5: The model consistently offers highly relevant and useful responses that perfectly cater to the user's needs and inquiries."""
+
+# Default Eval Prompt
+DEFAULT_EVAL_PROMPT = """Does the model provide relevant and useful responses to the user's needs or questions?
+
+Scoring Rubric:
+Score 1: The model's responses are irrelevant or unhelpful to the user's needs or queries.
+Score 2: The model sometimes provides helpful information, but often fails to address the user's actual needs or questions.
+Score 3: The model generally provides helpful responses that address the user's needs, though it may occasionally miss the mark.
+Score 4: The model regularly provides helpful responses that are well-aligned with the user's inquiries, with only rare inaccuracies.
+Score 5: The model consistently offers highly relevant and useful responses that perfectly cater to the user's needs and inquiries.
+
+[User Query]: {{input}}
+
+[AI Response]: {{response}}"""
+
+# Split the eval prompt into editable and fixed parts
+DEFAULT_EVAL_PROMPT_EDITABLE = """Does the model provide relevant and useful responses to the user's needs or questions?
+
+Scoring Rubric:
+Score 1: The model's responses are irrelevant or unhelpful to the user's needs or queries.
+Score 2: The model sometimes provides helpful information, but often fails to address the user's actual needs or questions.
+Score 3: The model generally provides helpful responses that address the user's needs, though it may occasionally miss the mark.
+Score 4: The model regularly provides helpful responses that are well-aligned with the user's inquiries, with only rare inaccuracies.
+Score 5: The model consistently offers highly relevant and useful responses that perfectly cater to the user's needs and inquiries."""
+
+# Fixed suffix that will always be appended
+FIXED_EVAL_SUFFIX = """
+[User Query]: {{human_input}}
+
+[AI Response]: {{ai_response}}"""
+
+ATLA_PROMPT = """You are tasked with evaluating a response based on a given instruction (which may contain an Input) and a scoring rubric that serve as the evaluation standard. Provide a comprehensive feedback on the response quality strictly adhering to the scoring rubric, without any general evaluation. Follow this with a score integer, referring to the scoring rubric. Avoid generating any additional opening, closing, or explanations.
+  Here are some rules of the evaluation:
+  (1) You should prioritize evaluating whether the response satisfies the provided rubric. The basis of your score should depend exactly on the rubric. However, the response does not need to explicitly address points raised in the rubric. Rather, evaluate the response based on the criteria outlined in the rubric.
+
+  Your reply should strictly follow this format:
+  **Reasoning:** <Your feedback>
+
+  **Result:** <Your score>
+
+  Here is the data:
+
+  Instruction:
+  ```
+  {human_input}
+  ```
+
+  Response:
+  ```
+  {ai_response}
+  ```
+
+  Score Rubrics:
+  {eval_criteria}"""
+
+ATLA_PROMPT_WITH_REFERENCE = """You are tasked with evaluating a response based on a given instruction (which may contain an Input) and a scoring rubric and reference answer that serve as the evaluation standard. Provide a comprehensive feedback on the response quality strictly adhering to the scoring rubric, without any general evaluation. Follow this with a score, referring to the scoring rubric. Avoid generating any additional opening, closing, or explanations.
+
+  Here are some rules of the evaluation:
+  (1) You should prioritize evaluating whether the response satisfies the provided rubric. The basis of your score should depend exactly on the rubric. However, the response does not need to explicitly address points raised in the rubric. Rather, evaluate the response based on the criteria outlined in the rubric.
+
+  Your reply should strictly follow this format:
+  **Reasoning:** <Your feedback>
+
+  **Result:** <Your score>
+
+  Here is the data:
+
+  Instruction:
+  ```
+  {human_input}
+  ```
+
+  Response:
+  ```
+  {ai_response}
+  ```
+
+  Score Rubrics:
+  {eval_criteria}
+
+  Reference answer:
+  {ground_truth}"""
+
+# Judge system prompt for non-Prometheus models
+JUDGE_SYSTEM_PROMPT = """Please act as an impartial judge and evaluate based on the user's instruction. Your output format should strictly adhere to JSON as follows: {"feedback": "<write feedback>", "result": <numerical score>}. Ensure the output is valid JSON, without additional formatting or explanations.""" 

random_sample/random_sample_generation.py

@@ -0,0 +1,183 @@
+from openai import OpenAI
+import anthropic
+import json
+import re
+import random
+import os
+from .gen_api_answer import get_openai_response, get_anthropic_response
+
+# Initialize clients
+anthropic_client = anthropic.Anthropic()
+openai_client = OpenAI()
+
+GOOD_SYSTEM_PROMPT = """You are an assistant that generates random conversations between a human and an AI assistant for testing purposes. The AI response generated should be a few sentences long. Format your output as JSON: {"human": "<human message>", "ai": <AI assistant response>}. Ensure the output is valid JSON, without additional formatting or explanations."""
+BAD_SYSTEM_PROMPT = """You are an assistant that generates random conversations between a human and an AI assistant for testing purposes. The response should contain incorrect information, logical fallacies, or misleading explanations. It should sound plausible but be fundamentally wrong. The AI response generated should be a few sentences long. Format your output as JSON: {"human": "<human message>", "ai": <AI assistant response>}. Ensure the output is valid JSON, without additional formatting or explanations."""
+AMBIGUOUS_SYSTEM_PROMPT = """You are an assistant that generates random conversations between a human and an AI assistant for testing purposes. The response should mix correct and incorrect information - it should contain some accurate points but also include nuanced, questionable claims or exaggerations. The AI response generated should be a few sentences long. Format your output as JSON: {"human": "<human message>", "ai": <AI assistant response>}. Ensure the output is valid JSON, without additional formatting or explanations."""
+
+GOOD_SYSTEM_PROMPT_WITH_GROUND_TRUTH = """You are an assistant that generates random conversations between a human and an AI assistant for testing purposes, along with an ideal reference answer. The AI response generated should be a few sentences long and contain accurate information. The ground truth response should be a perfect, comprehensive answer that would score 5/5. Format your output as JSON: {"human": "<human message>", "ai": "<AI assistant response>", "ground_truth": "<perfect reference answer>"}. Ensure the output is valid JSON, without additional formatting or explanations."""
+BAD_SYSTEM_PROMPT_WITH_GROUND_TRUTH = """You are an assistant that generates random conversations between a human and an AI assistant for testing purposes, along with an ideal reference answer. The AI response should be a few sentences long and contain incorrect information, logical fallacies, or misleading explanations. It should sound plausible but be fundamentally wrong. The ground truth response should be a perfect, comprehensive answer that would score 5/5. Format your output as JSON: {"human": "<human message>", "ai": "<AI assistant response>", "ground_truth": "<perfect reference answer>"}. Ensure the output is valid JSON, without additional formatting or explanations."""
+AMBIGUOUS_SYSTEM_PROMPT_WITH_GROUND_TRUTH = """You are an assistant that generates random conversations between a human and an AI assistant for testing purposes, along with an ideal reference answer. The AI response should be a few sentences long and mix correct and incorrect information - it should contain some accurate points but also include nuanced, questionable claims or exaggerations. The ground truth response should be a perfect, comprehensive answer that would score 5/5. Format your output as JSON: {"human": "<human message>", "ai": "<AI assistant response>", "ground_truth": "<perfect reference answer>"}. Ensure the output is valid JSON, without additional formatting or explanations."""
+
+GENERATION_PROMPT = """Please generate a random human message and an AI response in the format of a QA dataset. The human input should not be a one-word answer question like "What is the capital of France?". The AI response generated should be a few sentences long."""
+GENERATION_PROMPT_WITH_GROUND_TRUTH = """Please generate:
+1. A random human message (not a simple one-word answer question)
+2. An AI response (a few sentences long)
+3. A perfect reference answer that would score 5/5 on all criteria (e.g., concise, helpful, and accurate)
+
+Format as JSON with "human", "ai", and "ground_truth" fields."""
+
+RESPONSE_GENERATION_SYSTEM_PROMPT = "You are an assistant that generates random responses to human messages for testing purposes. Generate bad responses (with a mix of correct and incorrect information) 60% of the time and good responses 40% of the time. Do not say which type of response you are generating, just generate the response."
+
+def get_random_human_ai_pair():
+    # Select system prompt with specified probabilities
+    system_prompt = random.choices(
+        [GOOD_SYSTEM_PROMPT, BAD_SYSTEM_PROMPT, AMBIGUOUS_SYSTEM_PROMPT],
+        weights=[0.2, 0.2, 0.6]  # 20% good, 20% bad, 60% ambiguous
+    )[0]
+    
+    # Log which type of response is being generated
+    prompt_type = {
+        GOOD_SYSTEM_PROMPT: "good",
+        BAD_SYSTEM_PROMPT: "bad", 
+        AMBIGUOUS_SYSTEM_PROMPT: "ambiguous"
+    }[system_prompt]
+    print(f"Generating {prompt_type} response")
+    
+    # Randomly choose between GPT-3.5 and Claude with 65%/35% weights
+    model_choice = random.choices([
+        ("gpt-3.5-turbo", get_openai_response),
+        ("claude-3-5-haiku-latest", get_anthropic_response)
+    ], weights=[0.5, 0.5])[0]
+    model_name, api_func = model_choice
+    
+    # Generate response using selected model
+    response = api_func(
+        model_name=model_name,
+        prompt=GENERATION_PROMPT,
+        system_prompt=system_prompt,
+        max_tokens=500,
+        temperature=1
+    )
+    
+    # Define default messages
+    default_human = "How do muscles grow?"
+    default_ai = """Muscles grow through a process called skeletal muscle hypertrophy, which adds more myosin filaments to each muscle fiber, making the engine of the cell bigger and stronger over time. This is achieved through increased muscle tension and physical stress, breaking down muscle fiber. Muscle growth is also a direct consequence of resistance training and nutrition. People build muscle at different rates depending on their age, sex, and genetics, but muscle development significantly increases if exercise is done correctly and the body stores more protein through a process called protein synthesis."""
+    
+    try:
+        # Clean the response by replacing newlines with spaces
+        cleaned_response = response.replace('\n', ' ').replace('\r', '')
+        data = json.loads(cleaned_response)
+        
+        # Extract messages with fallbacks
+        human_message = data.get("human", default_human)
+        ai_message = data.get("ai", default_ai)
+        
+        # Debug logging
+        print(f"Parsed response: human='{human_message}', ai='{ai_message[:50]}...'")
+        
+    except Exception as e:
+        print(f"Failed to parse response: {str(e)}\n {response}")
+        human_message = default_human
+        ai_message = default_ai
+    
+    return human_message, ai_message
+
+def get_random_human_ai_ground_truth_pair():
+    # Select system prompt with specified probabilities
+    system_prompts = {
+        "good": GOOD_SYSTEM_PROMPT_WITH_GROUND_TRUTH,
+        "bad": BAD_SYSTEM_PROMPT_WITH_GROUND_TRUTH,
+        "ambiguous": AMBIGUOUS_SYSTEM_PROMPT_WITH_GROUND_TRUTH
+    }
+    
+    prompt_type = random.choices(
+        ["good", "bad", "ambiguous"],
+        weights=[0.2, 0.2, 0.6]  # 20% good, 20% bad, 60% ambiguous
+    )[0]
+    
+    system_prompt = system_prompts[prompt_type]
+    print(f"Generating {prompt_type} response with ground truth")
+    
+    # Randomly choose between GPT-3.5 and Claude with 50/50 weights
+    model_choice = random.choices([
+        ("gpt-3.5-turbo", get_openai_response),
+        ("claude-3-5-haiku-latest", get_anthropic_response)
+    ], weights=[0.5, 0.5])[0]
+    model_name, api_func = model_choice
+    
+    # Define default messages
+    defaults = {
+        "human": "How do muscles grow?",
+        "ai": """Muscles grow through a process called skeletal muscle hypertrophy, which adds more myosin filaments to each muscle fiber, making the engine of the cell bigger and stronger over time. This is achieved through increased muscle tension and physical stress, breaking down muscle fiber. Muscle growth is also a direct consequence of resistance training and nutrition. People build muscle at different rates depending on their age, sex, and genetics, but muscle development significantly increases if exercise is done correctly and the body stores more protein through a process called protein synthesis.""",
+        "ground_truth": """Muscle growth (hypertrophy) occurs through a complex biological process involving several key mechanisms:
+
+1. Mechanical Tension: Resistance training creates mechanical tension in muscle fibers, triggering molecular and cellular responses that promote growth.
+
+2. Metabolic Stress: The depletion of energy resources and accumulation of metabolic byproducts during exercise contributes to muscle growth signaling.
+
+3. Muscle Damage: Exercise-induced micro-damage to muscle fibers activates satellite cells, which help repair and build new muscle tissue.
+
+4. Protein Synthesis: After exercise, increased protein synthesis rates exceed protein breakdown, leading to net muscle protein accretion.
+
+5. Hormonal Response: Exercise triggers the release of growth-promoting hormones like testosterone, growth hormone, and IGF-1.
+
+6. Recovery: Adequate rest between training sessions allows for repair and growth, supported by proper nutrition, particularly protein intake (1.6-2.2g/kg/day).
+
+This process is influenced by factors including genetics, age, sex, nutrition, sleep quality, and training variables. Optimal muscle growth requires a structured resistance training program, adequate protein intake, sufficient calories, and proper recovery."""
+    }
+    
+    # Generate response using selected model
+    response = api_func(
+        model_name=model_name,
+        prompt=GENERATION_PROMPT_WITH_GROUND_TRUTH,
+        system_prompt=system_prompt,
+        max_tokens=1000,  # Increased token limit to accommodate ground truth
+        temperature=1
+    )
+    
+    # Parse the response to get all three components
+    try:
+        # Clean the response by replacing newlines with spaces
+        cleaned_response = response.replace('\n', ' ').replace('\r', '')
+        data = json.loads(cleaned_response)
+        
+        # Extract messages with fallbacks
+        human_message = data.get("human", defaults["human"])
+        ai_message = data.get("ai", defaults["ai"])
+        ground_truth = data.get("ground_truth", defaults["ground_truth"])
+        
+        # Debug logging
+        print(f"Parsed response: human='{human_message}', ai='{ai_message[:50]}...', ground_truth='{ground_truth[:50]}...'")
+        
+    except Exception as e:
+        print(f"Failed to parse response: {str(e)}\n {response}")
+        human_message = defaults["human"]
+        ai_message = defaults["ai"]
+        ground_truth = defaults["ground_truth"]
+    
+    return human_message, ai_message, ground_truth
+
+def generate_ai_response(human_msg):
+    """Generate AI response using GPT-3.5-turbo"""
+    if not human_msg.strip():
+        return "", False
+        
+    try:
+        response = get_openai_response(
+            "gpt-3.5-turbo", 
+            human_msg,
+            system_prompt=RESPONSE_GENERATION_SYSTEM_PROMPT,
+            max_tokens=1000,
+            temperature=1
+        )
+        # Extract just the response content since we don't need JSON format here
+        if isinstance(response, str):
+            # Clean up any JSON formatting if present
+            try:
+                data = json.loads(response)
+                response = data.get("content", response)
+            except json.JSONDecodeError:
+                pass
+        return response, False  # Return response and button interactive state
+    except Exception as e:
+        return f"Error generating response: {str(e)}", False

random_sample_tab.py

@@ -0,0 +1,6 @@
+import gradio as gr
+from random_sample.arena_interface import create_arena_interface
+
+def random_sample_tab():
+    with gr.TabItem("Random samples"):
+        return create_arena_interface() 

requirements.txt

@@ -1,8 +1,13 @@
+# Core dependencies
 pymongo
 gradio
 python-dotenv
 openai
 anthropic
 together
-cohere
-transformers
+atla
+transformers
+
+# Development dependencies
+black
+pytest

score_handler.py

@@ -0,0 +1,244 @@
+import gradio as gr
+import pandas as pd
+import numpy as np
+import tempfile
+import os
+
+def handle_analysis(df_state, model_selection_group, analyze_results_button):
+    with gr.Group(visible=False) as analysis_group:
+        gr.Markdown("## Analysis")
+
+        # Dropdown to select the accuracy measurement
+        accuracy_measurement_dropdown = gr.Dropdown(
+            choices=['Accuracy', 'Pearson Correlation'],
+            label='Select Evaluation Metric'
+        )
+
+        # We remove the two compare dropdowns and only keep ground truth
+        with gr.Row():
+            ground_truth_dropdown = gr.Dropdown(
+                choices=[],
+                label='Select True Label Column'
+            )
+
+        # Define two side-by-side boxes for results
+        with gr.Row():
+            judge_a_result = gr.Textbox(
+                label="Judge A Results",
+                lines=10,
+                interactive=False,
+                visible=False
+            )
+            judge_b_result = gr.Textbox(
+                label="Judge B Results",
+                lines=10,
+                interactive=False,
+                visible=False
+            )
+
+        # Move the JSON output below those textboxes and buttons
+        json_output = gr.File(label="Results .json", interactive=False, visible=False)
+
+        # Now place the row of buttons AFTER the json_output
+        with gr.Row():
+            back_to_results_button = gr.Button("← Back to Results")
+            calculate_button = gr.Button("Calculate")
+            download_button = gr.Button("Download Results as JSON")
+
+    # Show analysis group
+    def show_analysis_group():
+        df = df_state.value
+        if df is not None:
+            columns = df.columns.tolist()
+        else:
+            columns = []
+        # Now we only update ground_truth_dropdown
+        return (
+            gr.update(visible=True),         # analysis_group
+            gr.update(visible=False),        # model_selection_group
+            gr.update(choices=columns),      # ground_truth_dropdown
+        )
+
+    analyze_results_button.click(
+        fn=show_analysis_group,
+        inputs=[],
+        outputs=[
+            analysis_group,
+            model_selection_group,
+            ground_truth_dropdown  # only this one
+        ]
+    )
+
+    def back_to_results():
+        return (
+            gr.update(visible=False),  # Hide analysis_group
+            gr.update(visible=True),   # Show model_selection_group
+        )
+
+    back_to_results_button.click(
+        fn=back_to_results,
+        inputs=[],
+        outputs=[analysis_group, model_selection_group]
+    )
+
+    def calculate_multiple_accuracies(measurement, ground_truth_col, df_state):
+        # Update column names to match new format
+        col2_name = "score_selene"
+        # Get the non-selene score column (should be the other score_* column)
+        df = df_state.value
+        score_columns = [col for col in df.columns if col.startswith('score_') and col != 'score_selene']
+        col3_name = score_columns[0] if score_columns else None
+        
+        if df is None:
+            return (
+                gr.update(value="No DataFrame available.", visible=True),
+                gr.update(value="No DataFrame available.", visible=True)
+            )
+
+        # Check if user-supplied ground_truth_col is valid
+        missing_columns = [col for col in [ground_truth_col, col2_name, col3_name] if col not in df.columns]
+        if missing_columns:
+            msg = f"Selected columns not found in DataFrame: {', '.join(missing_columns)}."
+            return (
+                gr.update(value=msg, visible=True),
+                gr.update(value=msg, visible=True)
+            )
+
+        # Compare ground_truth_col with score_selene
+        result1 = calculate_accuracy(
+            measurement, ground_truth_col, col2_name,
+            df_state, compare_to_ground_truth=True
+        )
+        text_a = f"Comparison: '{ground_truth_col}' vs. 'Selene'\n{result1}"
+
+        # Compare ground_truth_col with the other model's score
+        result2 = calculate_accuracy(
+            measurement, ground_truth_col, col3_name,
+            df_state, compare_to_ground_truth=True
+        )
+        # Extract model name from column name for display
+        model_name = col3_name.replace('score_', '').replace('_', ' ').title()
+        text_b = f"Comparison: '{ground_truth_col}' vs. '{model_name}'\n{result2}"
+
+        return (
+            gr.update(value=text_a, visible=True),
+            gr.update(value=text_b, visible=True)
+        )
+
+    # Now the calculate_button only expects measurement, ground_truth_col, df_state
+    calculate_button.click(
+        fn=calculate_multiple_accuracies,
+        inputs=[
+            accuracy_measurement_dropdown,
+            ground_truth_dropdown,
+            df_state
+        ],
+        outputs=[judge_a_result, judge_b_result]
+    )
+
+    def create_json_download(df_state):
+        if df_state.value is None:
+            return gr.update(value=None, visible=True)
+        
+        json_str = df_state.value.to_json(orient='records', indent=2)
+        temp_dir = tempfile.gettempdir()
+        file_path = os.path.join(temp_dir, 'atla_custom_eval_results.json')
+        with open(file_path, 'w', encoding='utf-8') as f:
+            f.write(json_str)
+        return gr.update(value=file_path, visible=True)
+
+    download_button.click(
+        fn=create_json_download,
+        inputs=[df_state],
+        outputs=[json_output]
+    )
+
+# Helper functions
+
+def calculate_accuracy(measurement, col1, col2, df_state, compare_to_ground_truth=False):
+    df = df_state.value
+    # No changes here (function remains sacred as per your request)
+    if df is None:
+        return "No DataFrame available."
+    if col1 not in df.columns or col2 not in df.columns:
+        return "Selected columns not found in DataFrame."
+
+    results_df = pd.DataFrame()
+    if compare_to_ground_truth:
+        results_df['ground_truth'] = df[col1]
+        results_df['predicted'] = df[col2]
+    else:
+        results_df['extracted_winner'] = df[col1]
+        results_df['truth_result'] = df[col2]
+
+    if measurement == 'Accuracy':
+        result = process_pairwise_accuracy(results_df, compare_to_ground_truth)
+        output_text = (
+            f"Overall Accuracy: {result['overall_accuracy']}\n"
+            f"Number of NaNs: {result['num_extracted_nan']}"
+        )
+    elif measurement == 'Pearson Correlation':
+        result = process_single_rating_pearson_correlation(results_df, compare_to_ground_truth)
+        output_text = (
+            f"Pearson Correlation: {result['overall_pearson_correlation']}\n"
+            f"Number of NaNs: {result['num_extracted_nan']}"
+        )
+    else:
+        output_text = "Unknown measurement selected."
+
+    return output_text
+
+def process_pairwise_accuracy(results_df: pd.DataFrame, compare_to_ground_truth=False) -> dict:
+    # Compute 'results' column based on whether comparing to ground truth
+    if compare_to_ground_truth:
+        # NEW: convert both columns to float
+        results_df['ground_truth'] = results_df['ground_truth'].apply(convert_to_float_or_nan)
+        results_df['predicted'] = results_df['predicted'].apply(convert_to_float_or_nan)
+
+        results_df['results'] = results_df['ground_truth'] == results_df['predicted']
+        num_extracted_nan = int(results_df['predicted'].isna().sum())
+    else:
+        results_df['results'] = results_df['extracted_winner'] == results_df['truth_result']
+        num_extracted_nan = int(results_df['extracted_winner'].isna().sum())
+
+    overall_accuracy = results_df['results'].mean()
+
+    return {
+        "overall_accuracy": overall_accuracy,
+        "num_extracted_nan": num_extracted_nan,
+    }
+
+def process_single_rating_pearson_correlation(
+    results_df: pd.DataFrame, compare_to_ground_truth=False
+) -> dict:
+    if compare_to_ground_truth:
+        pred_col = 'predicted'
+        truth_col = 'ground_truth'
+    else:
+        pred_col = 'extracted_winner'
+        truth_col = 'truth_result'
+
+    results_df[pred_col] = results_df[pred_col].apply(convert_to_float_or_nan)
+    results_df[truth_col] = results_df[truth_col].apply(convert_to_float_or_nan)
+
+    numerical_results = results_df.dropna(subset=[pred_col, truth_col])
+
+    if len(numerical_results) == 0:
+        pearson_corr = np.nan
+    else:
+        pearson_corr = numerical_results[pred_col].corr(numerical_results[truth_col])
+
+    num_extracted_nan = int(results_df[pred_col].isna().sum())
+
+    return {
+        "overall_pearson_correlation": pearson_corr if not pd.isna(pearson_corr) else 0.0,
+        "num_extracted_nan": num_extracted_nan,
+    }
+
+def convert_to_float_or_nan(extracted_input):
+    if extracted_input is None or pd.isna(extracted_input):
+        return np.nan
+    try:
+        return float(extracted_input)
+    except ValueError:
+        return np.nan

ui_components.py

@@ -0,0 +1,4 @@
+# ui_components.py
+import gradio as gr
+
+save_prompt_button = gr.Button("Save Prompt", visible=False)

utils.py

@@ -2,6 +2,16 @@ from dataclasses import dataclass
 from datetime import datetime
 import logging
 
+def parse_variables(prompt):
+    import re
+    # Extract variables enclosed in double curly braces
+    variables = re.findall(r"{{(.*?)}}", prompt)
+    # Remove duplicates while preserving order
+    seen = set()
+    variables = [
+        x.strip() for x in variables if not (x.strip() in seen or seen.add(x.strip()))
+    ]
+    return variables
 
 def get_logger(sink_name: str = "core_utils") -> logging.Logger:
     logging.basicConfig(