|
""" Enhanced Multi-LLM Agent Evaluation Runner - CORRECTED VERSION""" |
|
import os |
|
import gradio as gr |
|
import requests |
|
import pandas as pd |
|
from langchain_core.messages import HumanMessage |
|
from veryfinal import build_graph |
|
|
|
|
|
DEFAULT_API_URL = "https://agents-course-unit4-scoring.hf.space" |
|
|
|
|
|
class EnhancedMultiLLMAgent: |
|
"""A multi-provider LangGraph agent with proper response handling.""" |
|
def __init__(self): |
|
print("Enhanced Multi-LLM Agent initialized.") |
|
try: |
|
self.graph = build_graph(provider="groq") |
|
print("Multi-LLM Graph built successfully.") |
|
except Exception as e: |
|
print(f"Error building graph: {e}") |
|
self.graph = None |
|
|
|
def __call__(self, question: str) -> str: |
|
print(f"Agent received question: {question[:100]}...") |
|
|
|
if self.graph is None: |
|
return "Error: Agent not properly initialized" |
|
|
|
|
|
state = { |
|
"messages": [HumanMessage(content=question)], |
|
"query": question, |
|
"agent_type": "", |
|
"final_answer": "", |
|
"perf": {}, |
|
"agno_resp": "" |
|
} |
|
|
|
config = {"configurable": {"thread_id": f"eval_{hash(question)}"}} |
|
|
|
try: |
|
result = self.graph.invoke(state, config) |
|
|
|
|
|
if isinstance(result, dict): |
|
|
|
if 'final_answer' in result and result['final_answer']: |
|
answer = result['final_answer'] |
|
|
|
elif 'messages' in result and result['messages']: |
|
last_message = result['messages'][-1] |
|
if hasattr(last_message, 'content'): |
|
answer = last_message.content |
|
else: |
|
answer = str(last_message) |
|
else: |
|
answer = str(result) |
|
else: |
|
answer = str(result) |
|
|
|
|
|
answer = answer.strip() |
|
|
|
|
|
if answer == question or answer.startswith(question): |
|
return "Information not available" |
|
|
|
|
|
if "FINAL ANSWER:" in answer: |
|
answer = answer.split("FINAL ANSWER:")[-1].strip() |
|
|
|
|
|
if not answer or len(answer.strip()) == 0: |
|
return "No answer generated" |
|
|
|
return answer |
|
|
|
except Exception as e: |
|
error_msg = f"Error: {str(e)}" |
|
print(error_msg) |
|
return error_msg |
|
|
|
def run_and_submit_all(profile: gr.OAuthProfile | None): |
|
"""Fetch questions, run agent, and submit answers.""" |
|
space_id = os.getenv("SPACE_ID") |
|
|
|
if profile: |
|
username = f"{profile.username}" |
|
print(f"User logged in: {username}") |
|
else: |
|
print("User not logged in.") |
|
return "Please Login to Hugging Face with the button.", None |
|
|
|
api_url = DEFAULT_API_URL |
|
questions_url = f"{api_url}/questions" |
|
submit_url = f"{api_url}/submit" |
|
|
|
|
|
try: |
|
agent = EnhancedMultiLLMAgent() |
|
if agent.graph is None: |
|
return "Error: Failed to initialize agent properly", None |
|
except Exception as e: |
|
print(f"Error instantiating agent: {e}") |
|
return f"Error initializing agent: {e}", None |
|
|
|
agent_code = f"https://huggingface.co/spaces/{space_id}/tree/main" if space_id else "No space ID available" |
|
print(f"Agent code URL: {agent_code}") |
|
|
|
|
|
print(f"Fetching questions from: {questions_url}") |
|
try: |
|
response = requests.get(questions_url, timeout=15) |
|
response.raise_for_status() |
|
questions_data = response.json() |
|
if not questions_data: |
|
print("Fetched questions list is empty.") |
|
return "Fetched questions list is empty or invalid format.", None |
|
print(f"Fetched {len(questions_data)} questions.") |
|
except Exception as e: |
|
print(f"Error fetching questions: {e}") |
|
return f"Error fetching questions: {e}", None |
|
|
|
|
|
results_log = [] |
|
answers_payload = [] |
|
print(f"Running Enhanced Multi-LLM agent on {len(questions_data)} questions...") |
|
|
|
for i, item in enumerate(questions_data): |
|
task_id = item.get("task_id") |
|
question_text = item.get("question") |
|
|
|
if not task_id or question_text is None: |
|
print(f"Skipping item with missing task_id or question: {item}") |
|
continue |
|
|
|
print(f"Processing question {i+1}/{len(questions_data)}: {task_id}") |
|
|
|
try: |
|
submitted_answer = agent(question_text) |
|
|
|
|
|
if submitted_answer == question_text or submitted_answer.startswith(question_text): |
|
submitted_answer = "Information not available" |
|
|
|
answers_payload.append({"task_id": task_id, "submitted_answer": submitted_answer}) |
|
results_log.append({ |
|
"Task ID": task_id, |
|
"Question": question_text[:100] + "..." if len(question_text) > 100 else question_text, |
|
"Submitted Answer": submitted_answer[:200] + "..." if len(submitted_answer) > 200 else submitted_answer |
|
}) |
|
except Exception as e: |
|
error_msg = f"AGENT ERROR: {e}" |
|
print(f"Error running agent on task {task_id}: {e}") |
|
answers_payload.append({"task_id": task_id, "submitted_answer": error_msg}) |
|
results_log.append({ |
|
"Task ID": task_id, |
|
"Question": question_text[:100] + "..." if len(question_text) > 100 else question_text, |
|
"Submitted Answer": error_msg |
|
}) |
|
|
|
if not answers_payload: |
|
print("Agent did not produce any answers to submit.") |
|
return "Agent did not produce any answers to submit.", pd.DataFrame(results_log) |
|
|
|
|
|
submission_data = {"username": username.strip(), "agent_code": agent_code, "answers": answers_payload} |
|
status_update = f"Enhanced Multi-LLM Agent finished. Submitting {len(answers_payload)} answers for user '{username}'..." |
|
print(status_update) |
|
|
|
|
|
print(f"Submitting {len(answers_payload)} answers to: {submit_url}") |
|
try: |
|
response = requests.post(submit_url, json=submission_data, timeout=60) |
|
response.raise_for_status() |
|
result_data = response.json() |
|
final_status = ( |
|
f"Submission Successful!\n" |
|
f"User: {result_data.get('username')}\n" |
|
f"Overall Score: {result_data.get('score', 'N/A')}% " |
|
f"({result_data.get('correct_count', '?')}/{result_data.get('total_attempted', '?')} correct)\n" |
|
f"Message: {result_data.get('message', 'No message received.')}" |
|
) |
|
print("Submission successful.") |
|
results_df = pd.DataFrame(results_log) |
|
return final_status, results_df |
|
except Exception as e: |
|
status_message = f"Submission Failed: {e}" |
|
print(status_message) |
|
results_df = pd.DataFrame(results_log) |
|
return status_message, results_df |
|
|
|
|
|
with gr.Blocks() as demo: |
|
gr.Markdown("# Enhanced Multi-LLM Agent - CORRECTED VERSION") |
|
gr.Markdown( |
|
""" |
|
**Instructions:** |
|
1. Log in to your Hugging Face account using the button below. |
|
2. Click 'Run Evaluation & Submit All Answers' to fetch questions, run your agent, submit answers, and see the score. |
|
|
|
**FIXES APPLIED:** |
|
- β
Proper response extraction from graph state |
|
- β
Prevention of question repetition as answer |
|
- β
Enhanced prompt engineering for better responses |
|
- β
Improved error handling and validation |
|
- β
Search-enhanced processing for information retrieval |
|
""" |
|
) |
|
|
|
gr.LoginButton() |
|
run_button = gr.Button("Run Evaluation & Submit All Answers", variant="primary") |
|
status_output = gr.Textbox(label="Run Status / Submission Result", lines=5, interactive=False) |
|
results_table = gr.DataFrame(label="Questions and Agent Answers", wrap=True) |
|
|
|
run_button.click( |
|
fn=run_and_submit_all, |
|
outputs=[status_output, results_table] |
|
) |
|
|
|
if __name__ == "__main__": |
|
print("\n" + "-"*30 + " Enhanced Multi-LLM Agent CORRECTED Starting " + "-"*30) |
|
demo.launch(debug=True, share=False) |
|
|