Update core/evaluation_engine.py

core/evaluation_engine.py

@@ -1,188 +1,118 @@
 # algoforge_prime/core/evaluation_engine.py
 import random
-import time
-import traceback
-# IMPORTANT: The following import is for a HYPOTHETICAL safe executor.
-# You would need to implement or find a robust sandboxing solution.
-# from .restricted_env_executor import execute_python_code_safely # Example
-
-from core.llm_clients import call_huggingface_api, call_gemini_api, LLMResponse # Absolute
-from prompts.system_prompts import get_system_prompt # Absolute
-from prompts.prompt_templates import format_critique_user_prompt # Absolute
-
-class EvaluationResult:
-    def __init__(self, score=0, critique_text="", passed_tests=0, total_tests=0, execution_summary=None, raw_llm_critique_response=None):
-        self.score = score  # Final combined score
-        self.critique_text = critique_text # LLM based critique + execution summary
-        self.passed_tests = passed_tests
-        self.total_tests = total_tests
-        self.execution_summary = execution_summary # Error or success message from code execution
-        self.raw_llm_critique_response = raw_llm_critique_response
-
-    def __str__(self): # For simple string representation if needed
-        return f"Score: {self.score}/10. Tests: {self.passed_tests}/{self.total_tests}. Summary: {self.execution_summary}. Critique: {self.critique_text[:100]}..."
-
-def _parse_score_from_llm_text(llm_text_output: str) -> int:
-    """Helper to parse 'Score: X/10' from LLM's textual output."""
-    score = 0 # Default if not found or unparsable
-    if not llm_text_output or not isinstance(llm_text_output, str):
-        return score
-
+from .llm_clients import call_huggingface_api, call_gemini_api, LLMResponse
+from ..prompts.system_prompts import get_system_prompt
+from ..prompts.prompt_templates import format_critique_user_prompt
+# Import our (simulated) safe executor
+from .safe_executor import execute_python_code_with_tests, ExecutionResult # Assuming it's in the same 'core' package
+
+class EvaluationResultOutput: # Renamed to avoid conflict with safe_executor.ExecutionResult
+    def __init__(self, combined_score=0, llm_critique_text="", execution_details: ExecutionResult = None, raw_llm_response=None):
+        self.combined_score = combined_score
+        self.llm_critique_text = llm_critique_text # LLM's qualitative assessment
+        self.execution_details = execution_details # Object from safe_executor
+        self.raw_llm_response = raw_llm_response
+
+    def get_display_critique(self):
+        full_critique = self.llm_critique_text if self.llm_critique_text else "LLM critique was not performed or failed."
+        if self.execution_details:
+            full_critique += f"\n\n**Automated Execution & Test Results (Simulated):**\n"
+            if self.execution_details.total_tests > 0:
+                full_critique += f"  Tests Attempted: {self.execution_details.total_tests}\n"
+                full_critique += f"  Tests Passed:    {self.execution_details.passed_tests}\n"
+            if self.execution_details.error:
+                full_critique += f"  Execution Error: {self.execution_details.error}\n"
+            elif self.execution_details.output:
+                 full_critique += f"  Execution Output (stdout):\n```\n{self.execution_details.output[:500]}\n```\n" # Limit output display
+            full_critique += f"  Execution Time: {self.execution_details.execution_time:.4f}s\n"
+        return full_critique
+
+
+def _parse_llm_score(llm_text_output: str) -> int:
+    # ... (keep your existing _parse_score_from_llm_text, renamed for clarity) ...
+    score = 0 
+    if not llm_text_output or not isinstance(llm_text_output, str): return score
     try:
-        # Look for "Score: X/10" or "Score: X"
-        # More robust parsing might be needed depending on LLM variability
         import re
         match = re.search(r"Score:\s*(\d+)(?:\s*/\s*10)?", llm_text_output, re.IGNORECASE)
         if match:
             parsed_score_val = int(match.group(1))
-            score = max(1, min(parsed_score_val, 10)) # Clamp score to 1-10
-        else: # Fallback if specific format not found
-            print(f"INFO: evaluation_engine.py - 'Score: X/10' marker not found in LLM output. Assigning fallback score. Output: {llm_text_output[:100]}...")
-            score = random.randint(3, 6) # Assign a mediocre random score
-    except Exception as e:
-        print(f"WARNING: evaluation_engine.py - Error parsing score from LLM output '{llm_text_output[:100]}...': {e}")
-        score = random.randint(3, 5) # Fallback on parsing error
+            score = max(1, min(parsed_score_val, 10)) 
+        else: score = random.randint(3, 6) # Fallback if no score marker
+    except Exception: score = random.randint(3, 5) # Fallback on any parsing error
     return score
 
-def _placeholder_safe_python_execution(code_string: str, user_tests_string: str) -> tuple[int, int, str]:
-    """
-    PLACEHOLDER for safe Python code execution.
-    **WARNING: THIS IS NOT SAFE FOR PRODUCTION. IT ONLY SIMULATES.**
-    Replace with a robust sandboxing mechanism (Docker, nsjail, WASM, etc.).
-    """
-    print(f"DEV_INFO: evaluation_engine.py - Entering PLACEHOLDER for code execution.")
-    print(f"  Code (first 100 chars): {code_string[:100]}...")
-    print(f"  Tests (first 100 chars): {user_tests_string[:100]}...")
-
-    if not user_tests_string.strip() or not code_string.strip():
-        return 0, 0, "SIMULATED: No tests provided or no code to test."
-
-    # Naive parsing of assert statements
-    test_lines = [line.strip() for line in user_tests_string.splitlines() if line.strip().startswith("assert")]
-    total_tests_found = len(test_lines)
-
-    if total_tests_found == 0:
-        return 0, 0, "SIMULATED: No 'assert' statements found in user tests."
-
-    # Extremely simplistic simulation logic (NOT REAL EXECUTION)
-    passed_count = 0
-    execution_log = ["SIMULATED EXECUTION LOG:"]
-    try:
-        # This is where real sandboxed execution would happen.
-        # We'll simulate based on keywords for demonstration.
-        if "syntax error" in code_string.lower() or "indentationerror" in code_string.lower():
-            execution_log.append("  - Simulated: Potential syntax error in generated code.")
-            # passed_count remains 0
-        elif "runtime error" in code_string.lower() or "exception" in code_string.lower():
-            execution_log.append("  - Simulated: Code might raise a runtime error.")
-            passed_count = random.randint(0, total_tests_found // 3) # Few pass
-        elif "return" not in code_string and any("==" in t for t in test_lines): # If expecting a return value
-            execution_log.append("  - Simulated: Code might be missing a crucial 'return' statement.")
-            passed_count = random.randint(0, total_tests_found // 2)
-        else: # Simulate some passing, some failing
-            passed_count = random.randint(total_tests_found // 2, total_tests_found)
-            execution_log.append(f"  - Simulated: {passed_count} of {total_tests_found} tests likely passed.")
-        
-        if passed_count < total_tests_found:
-            execution_log.append(f"  - Simulated: {total_tests_found - passed_count} test(s) likely failed.")
-        
-        summary = f"Simulated: {passed_count}/{total_tests_found} tests passed."
-        if passed_count < total_tests_found : summary += " Some tests likely failed."
-
-    except Exception as e_sim: # Error in our simulation logic
-        summary = f"Error during test SIMULATION logic: {str(e_sim)}"
-        passed_count = 0
-        execution_log.append(f"  - ERROR in simulation: {e_sim}")
-    
-    print(f"DEV_INFO: evaluation_engine.py - Placeholder execution result: {summary}")
-    return passed_count, total_tests_found, "\n".join(execution_log)
-
 
 def evaluate_solution_candidate(
     solution_text: str,
     problem_description: str,
     problem_type: str,
-    user_provided_tests: str, # String of Python assert statements
-    llm_client_config: dict # {"type": ..., "model_id": ..., "temp": ..., "max_tokens": ...}
-) -> EvaluationResult:
-    """
-    Evaluates a single solution candidate.
-    """
-    llm_critique_output_text = "LLM critique could not be performed due to an earlier error or API issue."
-    llm_based_score = 0
+    user_provided_tests_code: str,
+    llm_client_config: dict
+) -> EvaluationResultOutput:
+    
+    llm_critique_text = "LLM critique generation failed or was skipped."
+    llm_score = 0
     raw_llm_critique_resp = None
+    execution_result_obj = None # type: ExecutionResult
 
-    # 1. LLM-based Critique (if solution_text is not an error message itself)
+    # 1. LLM-based Critique (only if solution_text is not an error itself)
     if solution_text and not solution_text.startswith("ERROR"):
-        system_p_critique = get_system_prompt("critique_general") # problem_type can be used here too
+        system_p_critique = get_system_prompt("critique_general")
         user_p_critique = format_critique_user_prompt(problem_description, solution_text)
         
-        llm_response_obj = None # type: LLMResponse
+        llm_response_obj = None
         if llm_client_config["type"] == "hf":
-            llm_response_obj = call_huggingface_api(
-                user_p_critique, llm_client_config["model_id"],
-                temperature=llm_client_config["temp"], max_new_tokens=llm_client_config["max_tokens"],
-                system_prompt_text=system_p_critique
-            )
+            llm_response_obj = call_huggingface_api(user_p_critique, llm_client_config["model_id"], llm_client_config["temp"], llm_client_config["max_tokens"], system_p_critique)
         elif llm_client_config["type"] == "google_gemini":
-            llm_response_obj = call_gemini_api(
-                user_p_critique, llm_client_config["model_id"],
-                temperature=llm_client_config["temp"], max_new_tokens=llm_client_config["max_tokens"],
-                system_prompt_text=system_p_critique
-            )
+            llm_response_obj = call_gemini_api(user_p_critique, llm_client_config["model_id"], llm_client_config["temp"], llm_client_config["max_tokens"], system_p_critique)
         
         if llm_response_obj:
             raw_llm_critique_resp = llm_response_obj.raw_response
             if llm_response_obj.success:
-                llm_critique_output_text = llm_response_obj.text
-                llm_based_score = _parse_score_from_llm_text(llm_critique_output_text)
+                llm_critique_text = llm_response_obj.text
+                llm_score = _parse_llm_score(llm_critique_text)
             else:
-                llm_critique_output_text = f"Error during LLM critique (Model: {llm_response_obj.model_id_used}): {llm_response_obj.error}"
-                llm_based_score = 0 # Penalize for critique failure
+                llm_critique_text = f"Error during LLM critique (Model: {llm_response_obj.model_id_used}): {llm_response_obj.error}"
+                llm_score = 0 # Penalize
     elif solution_text and solution_text.startswith("ERROR"):
-        llm_critique_output_text = f"Solution was an error from Genesis: {solution_text}"
-        llm_based_score = 0
-
+        llm_critique_text = f"Solution was an error from Genesis: {solution_text}"
+        llm_score = 0
 
-    # 2. (Simulated) Code Execution if applicable
-    passed_tests_count = 0
-    total_tests_count = 0
-    exec_summary_msg = "Automated tests not applicable or not run for this problem type/solution."
 
-    # Only run tests if it's a Python problem, tests are provided, and solution isn't an error
-    if "python" in problem_type.lower() and user_provided_tests.strip() and solution_text and not solution_text.startswith("ERROR"):
-        # **IMPORTANT**: Replace with a REAL sandboxed executor for safety.
-        passed_tests_count, total_tests_count, exec_summary_msg = _placeholder_safe_python_execution(
-            solution_text, user_provided_tests
+    # 2. Code Execution (if Python problem, code exists, and tests are provided)
+    if "python" in problem_type.lower() and solution_text and not solution_text.startswith("ERROR") and user_provided_tests_code.strip():
+        print(f"INFO: evaluation_engine.py - Preparing to execute Python code candidate against user tests.")
+        # Use the (simulated) safe executor
+        execution_result_obj = execute_python_code_with_tests(
+            solution_text, user_provided_tests_code, timeout_seconds=10 # Example timeout
         )
-    elif "python" in problem_type.lower() and not user_provided_tests.strip():
-        exec_summary_msg = "No user tests provided for this Python problem."
-
-
-    # 3. Combine Scores into a Final Score (Example Heuristic)
-    final_score_calculated = llm_based_score
-    if total_tests_count > 0: # If tests were run
-        test_pass_ratio = passed_tests_count / total_tests_count
-        if test_pass_ratio < 0.5 : # Penalize heavily if less than half tests pass
-            final_score_calculated = max(1, int(llm_based_score * 0.5) - 1)
-        elif test_pass_ratio == 1.0 and passed_tests_count > 0: # All tests passed
-            final_score_calculated = min(10, llm_based_score + 1 if llm_based_score < 10 else 10) # Small bonus
-        else: # Some tests passed or ratio between 0.5 and 1.0
-            final_score_calculated = int(llm_based_score * (0.6 + 0.4 * test_pass_ratio))
-    final_score_calculated = max(1, min(10, final_score_calculated)) # Ensure score is 1-10
-
-    # Construct comprehensive critique text for display
-    comprehensive_critique = f"{llm_critique_output_text}"
-    if total_tests_count > 0 or ("python" in problem_type.lower() and user_provided_tests.strip()): # Add test summary if applicable
-        comprehensive_critique += f"\n\n**Automated Test Summary (Simulated):**\n{exec_summary_msg}\n"
-        comprehensive_critique += f"Passed: {passed_tests_count}/{total_tests_count}"
-
+        print(f"INFO: evaluation_engine.py - Execution result: {execution_result_obj}")
+    elif "python" in problem_type.lower() and not user_provided_tests_code.strip():
+         execution_result_obj = ExecutionResult(success=True, output="No user tests provided to run against the Python code.", total_tests=0)
+
+
+    # 3. Combine Scores into a Final Score (More sophisticated heuristic)
+    combined_score = llm_score
+    if execution_result_obj and execution_result_obj.total_tests > 0:
+        if not execution_result_obj.success or execution_result_obj.error: # Major execution failure
+            combined_score = max(1, llm_score - 5) # Penalize heavily
+        else:
+            pass_ratio = execution_result_obj.passed_tests / execution_result_obj.total_tests
+            if pass_ratio == 1.0: # All tests passed
+                combined_score = min(10, llm_score + 2) # Significant bonus
+            elif pass_ratio >= 0.75: # Most tests passed
+                combined_score = min(10, llm_score + 1) # Small bonus
+            elif pass_ratio < 0.25: # Very few tests passed
+                combined_score = max(1, llm_score - 4)
+            else: # Some tests passed
+                combined_score = int(llm_score * (0.5 + 0.5 * pass_ratio)) # Weighted average
+    
+    combined_score = max(1, min(10, combined_score)) # Clamp 1-10
 
-    return EvaluationResult(
-        score=final_score_calculated,
-        critique_text=comprehensive_critique,
-        passed_tests=passed_tests_count,
-        total_tests=total_tests_count,
-        execution_summary=exec_summary_msg,
-        raw_llm_critique_response=raw_llm_critique_resp
+    return EvaluationResultOutput(
+        combined_score=combined_score,
+        llm_critique_text=llm_critique_text,
+        execution_details=execution_result_obj,
+        raw_llm_response=raw_llm_critique_resp
     )