core/evaluation_engine.py

# algoforge_prime/core/evaluation_engine.py
import random
import time
import traceback
# IMPORTANT: The following import is for a HYPOTHETICAL safe executor.
# You would need to implement or find a robust sandboxing solution.
# from .restricted_env_executor import execute_python_code_safely # Example

from core.llm_clients import call_huggingface_api, call_gemini_api, LLMResponse # Absolute
from prompts.system_prompts import get_system_prompt # Absolute
from prompts.prompt_templates import format_critique_user_prompt # Absolute

class EvaluationResult:
    def __init__(self, score=0, critique_text="", passed_tests=0, total_tests=0, execution_summary=None, raw_llm_critique_response=None):
        self.score = score  # Final combined score
        self.critique_text = critique_text # LLM based critique + execution summary
        self.passed_tests = passed_tests
        self.total_tests = total_tests
        self.execution_summary = execution_summary # Error or success message from code execution
        self.raw_llm_critique_response = raw_llm_critique_response

    def __str__(self): # For simple string representation if needed
        return f"Score: {self.score}/10. Tests: {self.passed_tests}/{self.total_tests}. Summary: {self.execution_summary}. Critique: {self.critique_text[:100]}..."

def _parse_score_from_llm_text(llm_text_output: str) -> int:
    """Helper to parse 'Score: X/10' from LLM's textual output."""
    score = 0 # Default if not found or unparsable
    if not llm_text_output or not isinstance(llm_text_output, str):
        return score

    try:
        # Look for "Score: X/10" or "Score: X"
        # More robust parsing might be needed depending on LLM variability
        import re
        match = re.search(r"Score:\s*(\d+)(?:\s*/\s*10)?", llm_text_output, re.IGNORECASE)
        if match:
            parsed_score_val = int(match.group(1))
            score = max(1, min(parsed_score_val, 10)) # Clamp score to 1-10
        else: # Fallback if specific format not found
            print(f"INFO: evaluation_engine.py - 'Score: X/10' marker not found in LLM output. Assigning fallback score. Output: {llm_text_output[:100]}...")
            score = random.randint(3, 6) # Assign a mediocre random score
    except Exception as e:
        print(f"WARNING: evaluation_engine.py - Error parsing score from LLM output '{llm_text_output[:100]}...': {e}")
        score = random.randint(3, 5) # Fallback on parsing error
    return score

def _placeholder_safe_python_execution(code_string: str, user_tests_string: str) -> tuple[int, int, str]:
    """
    PLACEHOLDER for safe Python code execution.
    **WARNING: THIS IS NOT SAFE FOR PRODUCTION. IT ONLY SIMULATES.**
    Replace with a robust sandboxing mechanism (Docker, nsjail, WASM, etc.).
    """
    print(f"DEV_INFO: evaluation_engine.py - Entering PLACEHOLDER for code execution.")
    print(f"  Code (first 100 chars): {code_string[:100]}...")
    print(f"  Tests (first 100 chars): {user_tests_string[:100]}...")

    if not user_tests_string.strip() or not code_string.strip():
        return 0, 0, "SIMULATED: No tests provided or no code to test."

    # Naive parsing of assert statements
    test_lines = [line.strip() for line in user_tests_string.splitlines() if line.strip().startswith("assert")]
    total_tests_found = len(test_lines)

    if total_tests_found == 0:
        return 0, 0, "SIMULATED: No 'assert' statements found in user tests."

    # Extremely simplistic simulation logic (NOT REAL EXECUTION)
    passed_count = 0
    execution_log = ["SIMULATED EXECUTION LOG:"]
    try:
        # This is where real sandboxed execution would happen.
        # We'll simulate based on keywords for demonstration.
        if "syntax error" in code_string.lower() or "indentationerror" in code_string.lower():
            execution_log.append("  - Simulated: Potential syntax error in generated code.")
            # passed_count remains 0
        elif "runtime error" in code_string.lower() or "exception" in code_string.lower():
            execution_log.append("  - Simulated: Code might raise a runtime error.")
            passed_count = random.randint(0, total_tests_found // 3) # Few pass
        elif "return" not in code_string and any("==" in t for t in test_lines): # If expecting a return value
            execution_log.append("  - Simulated: Code might be missing a crucial 'return' statement.")
            passed_count = random.randint(0, total_tests_found // 2)
        else: # Simulate some passing, some failing
            passed_count = random.randint(total_tests_found // 2, total_tests_found)
            execution_log.append(f"  - Simulated: {passed_count} of {total_tests_found} tests likely passed.")
        
        if passed_count < total_tests_found:
            execution_log.append(f"  - Simulated: {total_tests_found - passed_count} test(s) likely failed.")
        
        summary = f"Simulated: {passed_count}/{total_tests_found} tests passed."
        if passed_count < total_tests_found : summary += " Some tests likely failed."

    except Exception as e_sim: # Error in our simulation logic
        summary = f"Error during test SIMULATION logic: {str(e_sim)}"
        passed_count = 0
        execution_log.append(f"  - ERROR in simulation: {e_sim}")
    
    print(f"DEV_INFO: evaluation_engine.py - Placeholder execution result: {summary}")
    return passed_count, total_tests_found, "\n".join(execution_log)


def evaluate_solution_candidate(
    solution_text: str,
    problem_description: str,
    problem_type: str,
    user_provided_tests: str, # String of Python assert statements
    llm_client_config: dict # {"type": ..., "model_id": ..., "temp": ..., "max_tokens": ...}
) -> EvaluationResult:
    """
    Evaluates a single solution candidate.
    """
    llm_critique_output_text = "LLM critique could not be performed due to an earlier error or API issue."
    llm_based_score = 0
    raw_llm_critique_resp = None

    # 1. LLM-based Critique (if solution_text is not an error message itself)
    if solution_text and not solution_text.startswith("ERROR"):
        system_p_critique = get_system_prompt("critique_general") # problem_type can be used here too
        user_p_critique = format_critique_user_prompt(problem_description, solution_text)
        
        llm_response_obj = None # type: LLMResponse
        if llm_client_config["type"] == "hf":
            llm_response_obj = call_huggingface_api(
                user_p_critique, llm_client_config["model_id"],
                temperature=llm_client_config["temp"], max_new_tokens=llm_client_config["max_tokens"],
                system_prompt_text=system_p_critique
            )
        elif llm_client_config["type"] == "google_gemini":
            llm_response_obj = call_gemini_api(
                user_p_critique, llm_client_config["model_id"],
                temperature=llm_client_config["temp"], max_new_tokens=llm_client_config["max_tokens"],
                system_prompt_text=system_p_critique
            )
        
        if llm_response_obj:
            raw_llm_critique_resp = llm_response_obj.raw_response
            if llm_response_obj.success:
                llm_critique_output_text = llm_response_obj.text
                llm_based_score = _parse_score_from_llm_text(llm_critique_output_text)
            else:
                llm_critique_output_text = f"Error during LLM critique (Model: {llm_response_obj.model_id_used}): {llm_response_obj.error}"
                llm_based_score = 0 # Penalize for critique failure
    elif solution_text and solution_text.startswith("ERROR"):
        llm_critique_output_text = f"Solution was an error from Genesis: {solution_text}"
        llm_based_score = 0


    # 2. (Simulated) Code Execution if applicable
    passed_tests_count = 0
    total_tests_count = 0
    exec_summary_msg = "Automated tests not applicable or not run for this problem type/solution."

    # Only run tests if it's a Python problem, tests are provided, and solution isn't an error
    if "python" in problem_type.lower() and user_provided_tests.strip() and solution_text and not solution_text.startswith("ERROR"):
        # **IMPORTANT**: Replace with a REAL sandboxed executor for safety.
        passed_tests_count, total_tests_count, exec_summary_msg = _placeholder_safe_python_execution(
            solution_text, user_provided_tests
        )
    elif "python" in problem_type.lower() and not user_provided_tests.strip():
        exec_summary_msg = "No user tests provided for this Python problem."


    # 3. Combine Scores into a Final Score (Example Heuristic)
    final_score_calculated = llm_based_score
    if total_tests_count > 0: # If tests were run
        test_pass_ratio = passed_tests_count / total_tests_count
        if test_pass_ratio < 0.5 : # Penalize heavily if less than half tests pass
            final_score_calculated = max(1, int(llm_based_score * 0.5) - 1)
        elif test_pass_ratio == 1.0 and passed_tests_count > 0: # All tests passed
            final_score_calculated = min(10, llm_based_score + 1 if llm_based_score < 10 else 10) # Small bonus
        else: # Some tests passed or ratio between 0.5 and 1.0
            final_score_calculated = int(llm_based_score * (0.6 + 0.4 * test_pass_ratio))
    final_score_calculated = max(1, min(10, final_score_calculated)) # Ensure score is 1-10

    # Construct comprehensive critique text for display
    comprehensive_critique = f"{llm_critique_output_text}"
    if total_tests_count > 0 or ("python" in problem_type.lower() and user_provided_tests.strip()): # Add test summary if applicable
        comprehensive_critique += f"\n\n**Automated Test Summary (Simulated):**\n{exec_summary_msg}\n"
        comprehensive_critique += f"Passed: {passed_tests_count}/{total_tests_count}"


    return EvaluationResult(
        score=final_score_calculated,
        critique_text=comprehensive_critique,
        passed_tests=passed_tests_count,
        total_tests=total_tests_count,
        execution_summary=exec_summary_msg,
        raw_llm_critique_response=raw_llm_critique_resp
    )