Update core/evaluation_engine.py

core/evaluation_engine.py

@@ -2,53 +2,80 @@
 import random
 import traceback
 
-# --- Corrected Imports ---
-from core.llm_clients import call_huggingface_api, call_gemini_api, LLMResponse 
-from prompts.system_prompts import get_system_prompt 
-from prompts.prompt_templates import format_critique_user_prompt 
-from .safe_executor import execute_python_code_with_tests, ExecutionResult # CORRECTED: Relative import
+from core.llm_clients import call_huggingface_api, call_gemini_api, LLMResponse
+from prompts.system_prompts import get_system_prompt
+from prompts.prompt_templates import format_critique_user_prompt
+from core.safe_executor import execute_python_code_with_tests, ExecutionResult, TestResult # Import new classes
 
 print("DEBUG: core.evaluation_engine - Imports successful")
 
-# ... (rest of the EvaluationResultOutput class, _parse_llm_score, _placeholder_safe_python_execution, 
-#      and evaluate_solution_candidate function as previously provided and corrected) ...
-# Ensure all that logic is present here. For brevity, I am not pasting it all again.
-# The key change is the import line for safe_executor above.
-
 class EvaluationResultOutput:
     def __init__(self, combined_score=0, llm_critique_text="", execution_details: ExecutionResult = None, raw_llm_response=None):
-        self.combined_score, self.llm_critique_text, self.execution_details, self.raw_llm_response = combined_score, llm_critique_text, execution_details, raw_llm_response
-    def get_display_critique(self):
-        # ... (implementation as before)
-        full_critique = self.llm_critique_text if self.llm_critique_text else "LLM critique failed/skipped."
+        self.combined_score = combined_score
+        self.llm_critique_text = llm_critique_text 
+        self.execution_details = execution_details 
+        self.raw_llm_response = raw_llm_response
+
+    def get_display_critique(self) -> str:
+        """Formats a comprehensive critique including LLM feedback and execution results."""
+        critique_parts = []
+        critique_parts.append(self.llm_critique_text if self.llm_critique_text else "LLM critique was not performed or failed.")
+
         if self.execution_details:
-            full_critique += f"\n\n**Automated Execution & Test Results (Simulated):**\n"
-            if self.execution_details.total_tests > 0: full_critique += f"  Tests: {self.execution_details.passed_tests}/{self.execution_details.total_tests} passed.\n"
-            if self.execution_details.error: full_critique += f"  Error: {self.execution_details.error}\n"
-            elif self.execution_details.output: full_critique += f"  Output:\n```\n{self.execution_details.output[:500]}\n```\n"
-            full_critique += f"  Time: {self.execution_details.execution_time:.4f}s\n"
-        return full_critique
+            exec_details = self.execution_details
+            critique_parts.append("\n\n**Automated Execution & Test Results (Simulated):**")
+            if exec_details.compilation_error:
+                critique_parts.append(f"  Compilation Error: {exec_details.compilation_error}")
+            elif exec_details.timeout_error:
+                critique_parts.append(f"  Execution Timed Out after {exec_details.execution_time:.2f}s.")
+            else:
+                if exec_details.total_tests > 0:
+                    critique_parts.append(f"  Tests Attempted: {exec_details.total_tests}")
+                    critique_parts.append(f"  Tests Passed:    {exec_details.passed_tests}")
+                    if exec_details.passed_tests < exec_details.total_tests:
+                        critique_parts.append("  Failed Tests Details:")
+                        for test_res in exec_details.individual_test_results:
+                            if not test_res.passed:
+                                critique_parts.append(f"    - Test: `{test_res.test_string[:70]}...`")
+                                if test_res.error_message:
+                                    critique_parts.append(f"      Error: {test_res.error_message[:100]}...")
+                else: # Code ran, but no assert-based tests provided/found
+                    critique_parts.append("  Code executed (no assert-based tests found/run).")
+
+                if exec_details.stdout:
+                    critique_parts.append(f"  Execution Stdout (truncated):\n```\n{exec_details.stdout[:300].strip()}\n```")
+                if exec_details.stderr and not any(not tr.passed for tr in exec_details.individual_test_results if tr.error_message): # Show general stderr if not already part of a test fail
+                    critique_parts.append(f"  Execution Stderr (general):\n```\n{exec_details.stderr[:300].strip()}\n```")
+            critique_parts.append(f"  Simulated Execution Time: {exec_details.execution_time:.4f}s")
+        return "\n".join(critique_parts)
 
 def _parse_llm_score(llm_text_output: str) -> int:
-    # ... (implementation as before)
+    # ... (same as your last working version)
     score = 0; import re
     if not llm_text_output or not isinstance(llm_text_output, str): return score
     match = re.search(r"Score:\s*(\d+)(?:\s*/\s*10)?", llm_text_output, re.IGNORECASE)
     if match: score = max(1, min(int(match.group(1)), 10))
-    else: score = random.randint(3, 6)
+    else: 
+        print(f"INFO: evaluation_engine.py - 'Score: X/10' marker not found. Output: {llm_text_output[:100]}...")
+        score = random.randint(3, 6) 
     return score
 
-# _placeholder_safe_python_execution remains in safe_executor.py, it's imported.
-
 def evaluate_solution_candidate(
-    solution_text: str, problem_description: str, problem_type: str,
-    user_provided_tests_code: str, llm_client_config: dict
+    solution_text: str,
+    problem_description: str,
+    problem_type: str,
+    user_provided_tests_code: str,
+    llm_client_config: dict
 ) -> EvaluationResultOutput:
-    # ... (implementation as before, ensuring it calls the imported execute_python_code_with_tests) ...
     print(f"DEBUG: evaluation_engine.py - Evaluating candidate. Problem type: {problem_type}")
-    llm_critique_text, llm_score, raw_llm_critique_resp, execution_result_obj = "LLM critique failed/skipped.", 0, None, None
+    llm_critique_text = "LLM critique generation failed or was skipped."
+    llm_score = 0
+    raw_llm_critique_resp = None
+    execution_result_obj = None # type: ExecutionResult
+
+    # 1. LLM-based Critique
     if solution_text and not solution_text.startswith("ERROR"):
-        # ... (LLM critique call logic) ...
+        # ... (LLM critique call logic - same as before) ...
         system_p_critique = get_system_prompt("critique_general")
         user_p_critique = format_critique_user_prompt(problem_description, solution_text)
         llm_response_obj = None
@@ -58,24 +85,46 @@ def evaluate_solution_candidate(
             raw_llm_critique_resp = llm_response_obj.raw_response
             if llm_response_obj.success: llm_critique_text, llm_score = llm_response_obj.text, _parse_llm_score(llm_response_obj.text)
             else: llm_critique_text, llm_score = f"Error during LLM critique: {llm_response_obj.error}", 0
-    elif solution_text and solution_text.startswith("ERROR"): llm_critique_text, llm_score = f"Solution was error: {solution_text}", 0
+    elif solution_text and solution_text.startswith("ERROR"):
+        llm_critique_text, llm_score = f"Solution was error from Genesis: {solution_text}", 0
 
-    if "python" in problem_type.lower() and solution_text and not solution_text.startswith("ERROR") and user_provided_tests_code.strip():
-        execution_result_obj = execute_python_code_with_tests(solution_text, user_provided_tests_code, timeout_seconds=10)
-    elif "python" in problem_type.lower() and not user_provided_tests_code.strip():
-        execution_result_obj = ExecutionResult(success=True, output="No user tests provided.", total_tests=0)
-    
-    combined_score = llm_score # Start with LLM score
-    if execution_result_obj and execution_result_obj.total_tests > 0: # Adjust based on tests
-        if not execution_result_obj.success or execution_result_obj.error: combined_score = max(1, llm_score - 5)
+    # 2. Code Execution
+    if "python" in problem_type.lower() and solution_text and not solution_text.startswith("ERROR"):
+        if user_provided_tests_code.strip():
+            print(f"INFO: evaluation_engine.py - Executing Python code candidate against user tests.")
+            execution_result_obj = execute_python_code_with_tests(solution_text, user_provided_tests_code, timeout_seconds=10)
         else:
+            print(f"INFO: evaluation_engine.py - Executing Python code candidate (no tests provided).")
+            execution_result_obj = execute_python_code_with_tests(solution_text, "", timeout_seconds=5) # Execute code even if no tests
+        print(f"INFO: evaluation_engine.py - Execution result: {execution_result_obj}")
+    elif "python" in problem_type.lower() and not user_provided_tests_code.strip() and solution_text and not solution_text.startswith("ERROR"):
+        # Case where it's python but no tests - still might want to run to catch basic runtime/compile errors
+        execution_result_obj = execute_python_code_with_tests(solution_text, "", timeout_seconds=5)
+
+
+    # 3. Combine Scores into a Final Score
+    combined_score = llm_score
+    if execution_result_obj:
+        if execution_result_obj.compilation_error or execution_result_obj.timeout_error or (not execution_result_obj.success and execution_result_obj.stderr and not execution_result_obj.individual_test_results) :
+            combined_score = 1 # Catastrophic failure
+        elif execution_result_obj.total_tests > 0:
             pass_ratio = execution_result_obj.passed_tests / execution_result_obj.total_tests
-            if pass_ratio == 1.0: combined_score = min(10, llm_score + 2)
-            elif pass_ratio >= 0.75: combined_score = min(10, llm_score + 1)
-            elif pass_ratio < 0.25: combined_score = max(1, llm_score - 4)
-            else: combined_score = int(llm_score * (0.5 + 0.5 * pass_ratio))
+            if pass_ratio == 1.0: combined_score = min(10, llm_score + 3) # Strong bonus for all tests passing
+            elif pass_ratio >= 0.8: combined_score = min(10, llm_score + 1)
+            elif pass_ratio < 0.2: combined_score = max(1, llm_score - 6) # Heavy penalty
+            elif pass_ratio < 0.5: combined_score = max(1, llm_score - 4)
+            else: combined_score = int(llm_score * (0.4 + 0.6 * pass_ratio)) # Weighted more by tests
+        elif not execution_result_obj.success and execution_result_obj.error : # General runtime error without tests
+             combined_score = max(1, llm_score - 4)
+
     combined_score = max(1, min(10, combined_score))
-    return EvaluationResultOutput(combined_score, llm_critique_text, execution_result_obj, raw_llm_critique_resp)
 
+    print(f"DEBUG: evaluation_engine.py - Evaluation complete. Combined Score: {combined_score}")
+    return EvaluationResultOutput(
+        combined_score=combined_score,
+        llm_critique_text=llm_critique_text,
+        execution_details=execution_result_obj,
+        raw_llm_response=raw_llm_critique_resp
+    )
 
 print("DEBUG: core.evaluation_engine - Module fully defined.")