core/evaluation_engine.py

# algoforge_prime/core/evaluation_engine.py
import random
import traceback

from core.llm_clients import call_huggingface_api, call_gemini_api, LLMResponse
from prompts.system_prompts import get_system_prompt
from prompts.prompt_templates import format_critique_user_prompt
from core.safe_executor import execute_python_code_with_tests, ExecutionResult, TestResult # Import new classes

print("DEBUG: core.evaluation_engine - Imports successful")

class EvaluationResultOutput:
    def __init__(self, combined_score=0, llm_critique_text="", execution_details: ExecutionResult = None, raw_llm_response=None):
        self.combined_score = combined_score
        self.llm_critique_text = llm_critique_text 
        self.execution_details = execution_details 
        self.raw_llm_response = raw_llm_response

    def get_display_critique(self) -> str:
        """Formats a comprehensive critique including LLM feedback and execution results."""
        critique_parts = []
        critique_parts.append(self.llm_critique_text if self.llm_critique_text else "LLM critique was not performed or failed.")

        if self.execution_details:
            exec_details = self.execution_details
            critique_parts.append("\n\n**Automated Execution & Test Results (Simulated):**")
            if exec_details.compilation_error:
                critique_parts.append(f"  Compilation Error: {exec_details.compilation_error}")
            elif exec_details.timeout_error:
                critique_parts.append(f"  Execution Timed Out after {exec_details.execution_time:.2f}s.")
            else:
                if exec_details.total_tests > 0:
                    critique_parts.append(f"  Tests Attempted: {exec_details.total_tests}")
                    critique_parts.append(f"  Tests Passed:    {exec_details.passed_tests}")
                    if exec_details.passed_tests < exec_details.total_tests:
                        critique_parts.append("  Failed Tests Details:")
                        for test_res in exec_details.individual_test_results:
                            if not test_res.passed:
                                critique_parts.append(f"    - Test: `{test_res.test_string[:70]}...`")
                                if test_res.error_message:
                                    critique_parts.append(f"      Error: {test_res.error_message[:100]}...")
                else: # Code ran, but no assert-based tests provided/found
                    critique_parts.append("  Code executed (no assert-based tests found/run).")

                if exec_details.stdout:
                    critique_parts.append(f"  Execution Stdout (truncated):\n```\n{exec_details.stdout[:300].strip()}\n```")
                if exec_details.stderr and not any(not tr.passed for tr in exec_details.individual_test_results if tr.error_message): # Show general stderr if not already part of a test fail
                    critique_parts.append(f"  Execution Stderr (general):\n```\n{exec_details.stderr[:300].strip()}\n```")
            critique_parts.append(f"  Simulated Execution Time: {exec_details.execution_time:.4f}s")
        return "\n".join(critique_parts)

def _parse_llm_score(llm_text_output: str) -> int:
    # ... (same as your last working version)
    score = 0; import re
    if not llm_text_output or not isinstance(llm_text_output, str): return score
    match = re.search(r"Score:\s*(\d+)(?:\s*/\s*10)?", llm_text_output, re.IGNORECASE)
    if match: score = max(1, min(int(match.group(1)), 10))
    else: 
        print(f"INFO: evaluation_engine.py - 'Score: X/10' marker not found. Output: {llm_text_output[:100]}...")
        score = random.randint(3, 6) 
    return score

def evaluate_solution_candidate(
    solution_text: str,
    problem_description: str,
    problem_type: str,
    user_provided_tests_code: str,
    llm_client_config: dict
) -> EvaluationResultOutput:
    print(f"DEBUG: evaluation_engine.py - Evaluating candidate. Problem type: {problem_type}")
    llm_critique_text = "LLM critique generation failed or was skipped."
    llm_score = 0
    raw_llm_critique_resp = None
    execution_result_obj = None # type: ExecutionResult

    # 1. LLM-based Critique
    if solution_text and not solution_text.startswith("ERROR"):
        # ... (LLM critique call logic - same as before) ...
        system_p_critique = get_system_prompt("critique_general")
        user_p_critique = format_critique_user_prompt(problem_description, solution_text)
        llm_response_obj = None
        if llm_client_config["type"] == "hf": llm_response_obj = call_huggingface_api(user_p_critique, llm_client_config["model_id"], llm_client_config["temp"], llm_client_config["max_tokens"], system_p_critique)
        elif llm_client_config["type"] == "google_gemini": llm_response_obj = call_gemini_api(user_p_critique, llm_client_config["model_id"], llm_client_config["temp"], llm_client_config["max_tokens"], system_p_critique)
        if llm_response_obj:
            raw_llm_critique_resp = llm_response_obj.raw_response
            if llm_response_obj.success: llm_critique_text, llm_score = llm_response_obj.text, _parse_llm_score(llm_response_obj.text)
            else: llm_critique_text, llm_score = f"Error during LLM critique: {llm_response_obj.error}", 0
    elif solution_text and solution_text.startswith("ERROR"):
        llm_critique_text, llm_score = f"Solution was error from Genesis: {solution_text}", 0

    # 2. Code Execution
    if "python" in problem_type.lower() and solution_text and not solution_text.startswith("ERROR"):
        if user_provided_tests_code.strip():
            print(f"INFO: evaluation_engine.py - Executing Python code candidate against user tests.")
            execution_result_obj = execute_python_code_with_tests(solution_text, user_provided_tests_code, timeout_seconds=10)
        else:
            print(f"INFO: evaluation_engine.py - Executing Python code candidate (no tests provided).")
            execution_result_obj = execute_python_code_with_tests(solution_text, "", timeout_seconds=5) # Execute code even if no tests
        print(f"INFO: evaluation_engine.py - Execution result: {execution_result_obj}")
    elif "python" in problem_type.lower() and not user_provided_tests_code.strip() and solution_text and not solution_text.startswith("ERROR"):
        # Case where it's python but no tests - still might want to run to catch basic runtime/compile errors
        execution_result_obj = execute_python_code_with_tests(solution_text, "", timeout_seconds=5)


    # 3. Combine Scores into a Final Score
    combined_score = llm_score
    if execution_result_obj:
        if execution_result_obj.compilation_error or execution_result_obj.timeout_error or (not execution_result_obj.success and execution_result_obj.stderr and not execution_result_obj.individual_test_results) :
            combined_score = 1 # Catastrophic failure
        elif execution_result_obj.total_tests > 0:
            pass_ratio = execution_result_obj.passed_tests / execution_result_obj.total_tests
            if pass_ratio == 1.0: combined_score = min(10, llm_score + 3) # Strong bonus for all tests passing
            elif pass_ratio >= 0.8: combined_score = min(10, llm_score + 1)
            elif pass_ratio < 0.2: combined_score = max(1, llm_score - 6) # Heavy penalty
            elif pass_ratio < 0.5: combined_score = max(1, llm_score - 4)
            else: combined_score = int(llm_score * (0.4 + 0.6 * pass_ratio)) # Weighted more by tests
        elif not execution_result_obj.success and execution_result_obj.error : # General runtime error without tests
             combined_score = max(1, llm_score - 4)

    combined_score = max(1, min(10, combined_score))

    print(f"DEBUG: evaluation_engine.py - Evaluation complete. Combined Score: {combined_score}")
    return EvaluationResultOutput(
        combined_score=combined_score,
        llm_critique_text=llm_critique_text,
        execution_details=execution_result_obj,
        raw_llm_response=raw_llm_critique_resp
    )

print("DEBUG: core.evaluation_engine - Module fully defined.")