import os import chromadb from chromadb.utils import embedding_functions import json import re from openai import OpenAI import re import json import spacy from transformers import T5ForConditionalGeneration, T5Tokenizer from transformers import pipeline def robust_json_extractor(response_content): # Preprocess: Remove markdown code blocks and extra whitespace cleaned = re.sub(r'``````', '', response_content).strip() # Key-specific regex patterns patterns = { "verdict": r'"verdict"\s*:\s*"((?:\\"|[^"])*)"', "evidence": r'"evidence"\s*:\s*(\[[^\]]*?\]|\[.*?\])(?=\s*[,}])', "reasoning": r'"reasoning"\s*:\s*"((?:\\"|[^"])*)"' } result = {} for key, pattern in patterns.items(): match = re.search(pattern, cleaned, re.DOTALL) if match: try: if key == "evidence": # Handle array parsing with json.loads evidence_str = re.sub(r'(? 0] return [text] def verify_single_claim(self, claim, confidence_threshold=0.5): results = self.collection.query( query_texts=[claim], n_results=3, include=["documents", "metadatas", "distances"] ) zipped_results = sorted( zip(results['documents'][0], results['metadatas'][0], results['distances'][0]), key=lambda x: x[2] ) evidence = [] for doc, meta, distance in zipped_results: source = meta["source"] if meta and "source" in meta else "Unknown source" similarity_score = 1 - (distance / 2) # Assuming cosine distance in [0,2] evidence.append( f'"{doc}" (Source: {source}, Similarity: {similarity_score:.2f})' ) avg_distance = sum(d for _, _, d in zipped_results) / len(zipped_results) confidence = 1 - (avg_distance / 2) # Normalize to 0-1 range if confidence < confidence_threshold: return { "verdict": "Unverifiable", "confidence": confidence, "evidence": [e.split(" (Source:")[0] for e in evidence], "reasoning": "Claim is too vague or lacks sufficient evidence" } evidence_str = "\n".join([f"- {e}" for e in evidence]) prompt = f""" You are a powerful fact checker. Analyze the claim below against the provided verified information. Relying on the similarity scores, also carefully check whether all factual details in the claim (such as dates, names, locations, and events) exactly match atleast one of the evidence. If from first evidence, evidence is not sufficient, use the next evidence to verify the claim. If there is any factual mismatch (for example, the date in the claim is different from the evidence), classify the claim as False. Any factual mismatch, even if the overall context is similar, should lead to a False classification. If the evidence is too vague or lacks strong matches, classify as Unverifiable. If evidence directly contradicts the claim, classify as False. Any discrepancy in factual details, even if the overall context is similar, should lead to a False classification. If the evidence fully supports the claim with all factual details matching, classify as True. Claim: {claim} Evidence (with similarity scores): {evidence_str} Guidelines: 1. Give more weight to evidence with higher similarity scores, but do not ignore factual mismatches. 2. If any one piece of evidence independently supports the claim, without factual mismatches, classify as True. 2. Pay close attention to details such as dates, names, locations, and events. 3. If the claim and evidence differ on any factual point, do not classify as True. 4. Respond only in JSON format without any additional text. 5. In the "evidence" array, include only full evidence statements as strings, without any extra comments or explanations. 6. Put all explanations or comparisons in the "reasoning" field. Respond in JSON format: {{ "verdict": "Verdict", "evidence": [List of relevant facts from provided evidence], "reasoning": "Explanation of the verdict based on evidence and factual details" }} """ completion = self.groq_client.chat.completions.create( model=self.model_name, messages=[{"role": "user", "content": prompt}], temperature=0.1, max_tokens=400 ) response_content = completion.choices[0].message.content parsed = robust_json_extractor(response_content) if "error" in parsed: return { "error": parsed["error"], "confidence": confidence, "raw_response": parsed.get("raw", response_content) } else: required_keys = ["verdict", "evidence", "reasoning"] if all(key in parsed for key in required_keys): return { "verdict": parsed["verdict"], "confidence": confidence, "evidence": [e.split(" (Source:")[0] for e in evidence], "reasoning": parsed["reasoning"] } else: return { "error": f"Missing required keys: {[k for k in required_keys if k not in parsed]}", "confidence": confidence, "raw_response": response_content } def verify_single_entity(self, entity_text, confidence_threshold=0.5): """Verify a single named entity against the fact database""" # Vector similarity search results = self.collection.query( query_texts=[entity_text], n_results=3, include=["documents", "metadatas", "distances"] ) # Process evidence with similarity normalization evidence = [] total_distance = 0 for doc, meta, distance in zip(results['documents'][0], results['metadatas'][0], results['distances'][0]): similarity = 1 - (distance / 2) # Convert cosine distance to similarity evidence.append({ "text": doc, "source": meta.get("source", "Unknown"), "similarity": similarity }) total_distance += distance avg_similarity = 1 - (total_distance / len(results['distances'][0]) / 2) # Prepare LLM verification prompt evidence_str = "\n".join([ f"- {e['text']} (Similarity: {e['similarity']:.2f})" for e in evidence ]) prompt = f"""**Entity Verification Task** Entity: "{entity_text}" **Verified Evidence:** {evidence_str} **Instructions:** 1. Verify if this entity exists in official records 2. Check for exact matches of names/titles 3. Confirm associated details (locations, dates, roles) 4. Return JSON with: verdict (True/False/Unverified), confidence (0-1), reasoning **JSON Response:""" try: response = self.groq_client.chat.completions.create( model=self.model_name, messages=[{"role": "user", "content": prompt}], temperature=0.2, response_format={"type": "json_object"} ) result = json.loads(response.choices[0].message.content) return { "verdict": result.get("verdict", "Unverified"), "confidence": min(max(result.get("confidence", avg_similarity), 0), 1), "evidence": [e["text"] for e in evidence], "reasoning": result.get("reasoning", "No reasoning provided") } except Exception as e: return { "verdict": "Error", "confidence": 0, "evidence": [], "reasoning": f"Verification failed: {str(e)}" } def verify_claim(self, text, confidence_threshold=0.5): """ Main method: takes input text, extracts entities and claims, verifies each, and returns JSON results """ # Extract entities and claims entities = self.extract_entities(text) claims = self.extract_claims(text) # Verify claims claim_results = [] for claim in claims: verification = self.verify_single_claim(claim, confidence_threshold) claim_results.append({ "claim": claim, "verdict": verification.get("verdict", "Error"), "confidence": verification.get("confidence", 0), "evidence": verification.get("evidence", []), "reasoning": verification.get("reasoning", "Analysis failed") }) # Verify entities entity_results = [] for entity_text, entity_label in entities: verification = self.verify_single_entity(entity_text, confidence_threshold) entity_results.append({ "entity": entity_text, "type": entity_label, "verdict": verification.get("verdict", "Error"), "confidence": verification.get("confidence", 0), "evidence": verification.get("evidence", []), "reasoning": verification.get("reasoning", "Analysis failed") }) return { "entities": entity_results, "claims": claim_results }