Update main.py

main.py

@@ -447,11 +447,25 @@ class QualityAssessor:
         field_scores = result.confidence_scores.copy()
         consistency_score = self._check_consistency(result.data)
         
-        overall_confidence = (
-            sum(field_scores.values()) / len(field_scores) if field_scores else 0
-        ) * schema_compliance * consistency_score
+        required_fields = schema.get('required', [])
+        
+        if field_scores:
+            total_weight = 0
+            weighted_confidence = 0
+            
+            for field, confidence in field_scores.items():
+                weight = 2.0 if field in required_fields else 1.0
+                weighted_confidence += confidence * weight
+                total_weight += weight
+            
+            avg_field_confidence = weighted_confidence / total_weight
+        else:
+            avg_field_confidence = 0
         
-        review_flags = self._generate_review_flags(field_scores, schema_compliance, overall_confidence)
+        overall_confidence = avg_field_confidence * (0.8 + 0.2 * schema_compliance) * (0.9 + 0.1 * consistency_score)
+        overall_confidence = min(overall_confidence, 1.0)
+        
+        review_flags = self._generate_review_flags(field_scores, schema_compliance, overall_confidence, required_fields, result.data)
         review_time = self._estimate_review_time(review_flags, field_scores)
         
         return QualityReport(
@@ -467,19 +481,21 @@ class QualityAssessor:
         required_fields = schema.get('required', [])
         properties = schema.get('properties', {})
         
-        score = 1.0
-        
-        for field in required_fields:
-            if field not in data or data[field] is None:
-                score -= 0.2
+        required_present = sum(1 for field in required_fields if field in data and data[field] is not None)
+        required_compliance = required_present / len(required_fields) if required_fields else 1.0
         
+        type_errors = 0
+        total_fields = 0
         for field, value in data.items():
             if field in properties:
+                total_fields += 1
                 expected_type = properties[field].get('type')
                 if expected_type and not self._check_type(value, expected_type):
-                    score -= 0.1
+                    type_errors += 1
         
-        return max(0.0, score)
+        type_compliance = 1.0 - (type_errors / total_fields) if total_fields > 0 else 1.0
+        
+        return (required_compliance * 0.7 + type_compliance * 0.3)
     
     def _check_type(self, value: Any, expected_type: str) -> bool:
         if value is None:
@@ -497,20 +513,49 @@ class QualityAssessor:
         return isinstance(value, expected_python_type)
     
     def _check_consistency(self, data: Dict[str, Any]) -> float:
-        return 0.85
+        consistency_score = 1.0
+        
+        if 'email' in data and data['email']:
+            if '@' not in str(data['email']):
+                consistency_score -= 0.1
+        
+        if 'startDate' in data and 'endDate' in data:
+            try:
+                if data['startDate'] and data['endDate']:
+                    if str(data['startDate']) > str(data['endDate']):
+                        consistency_score -= 0.15
+            except:
+                pass
+        
+        if isinstance(data, dict):
+            for key, value in data.items():
+                if isinstance(value, list):
+                    for item in value:
+                        if isinstance(item, dict):
+                            consistency_score *= self._check_consistency(item)
+                elif isinstance(value, dict):
+                    consistency_score *= self._check_consistency(value)
+        
+        return max(0.7, consistency_score)
     
-    def _generate_review_flags(self, field_scores: Dict[str, float], schema_compliance: float, overall_confidence: float) -> List[str]:
+    def _generate_review_flags(self, field_scores: Dict[str, float], schema_compliance: float, overall_confidence: float, required_fields: List[str], extracted_data: Dict[str, Any]) -> List[str]:
         flags = []
         
-        if overall_confidence < 0.7:
-            flags.append("low_overall_confidence")
+        if overall_confidence < 0.6:
+            flags.append("high_priority_review")
+        elif overall_confidence < 0.8:
+            flags.append("standard_review")
         
         if schema_compliance < 0.8:
             flags.append("schema_compliance_issues")
         
-        low_confidence_fields = [field for field, score in field_scores.items() if score < 0.6]
+        low_confidence_fields = [field for field, score in field_scores.items() if score < 0.7]
         if low_confidence_fields:
-            flags.append(f"low_confidence_fields: {', '.join(low_confidence_fields)}")
+            flags.append(f"uncertain_fields: {', '.join(low_confidence_fields[:3])}")
+        
+        missing_required = [field for field in required_fields if field not in extracted_data or extracted_data[field] is None]
+        if missing_required:
+            flags.append(f"missing_required: {', '.join(missing_required[:3])}")
         
         return flags