File size: 7,090 Bytes
11cc0d3 91ebeb2 11cc0d3 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 |
import os
import chromadb
from chromadb.utils import embedding_functions
import json
import re
from openai import OpenAI
import re
import json
def robust_json_extractor(response_content):
# Preprocess: Remove markdown code blocks and extra whitespace
cleaned = re.sub(r'``````', '', response_content).strip()
# Key-specific regex patterns
patterns = {
"verdict": r'"verdict"\s*:\s*"((?:\\"|[^"])*)"',
"evidence": r'"evidence"\s*:\s*(\[[^\]]*?\]|\[.*?\])(?=\s*[,}])',
"reasoning": r'"reasoning"\s*:\s*"((?:\\"|[^"])*)"'
}
result = {}
for key, pattern in patterns.items():
match = re.search(pattern, cleaned, re.DOTALL)
if match:
try:
if key == "evidence":
# Handle array parsing with json.loads
evidence_str = re.sub(r'(?<!\\)"', r'\"', match.group(1)) # Escape unescaped quotes
result[key] = json.loads(evidence_str)
else:
# Unescape quotes for strings
result[key] = json.loads(f'"{match.group(1)}"')
except:
# Fallback: Return raw matched string
result[key] = match.group(1)
# Validation
required_keys = ["verdict", "evidence", "reasoning"]
if all(key in result for key in required_keys):
return result
else:
# Fallback to standard JSON parsing
try:
return json.loads(re.search(r'\{.*\}', cleaned, re.DOTALL).group())
except:
return {"error": "Failed to extract required keys", "raw": cleaned}
class FactChecker:
def __init__(self, chroma_path, collection_name, groq_client):
self.client = chromadb.PersistentClient(path=chroma_path)
self.collection = self.client.get_collection(
name=collection_name,
embedding_function=embedding_functions.SentenceTransformerEmbeddingFunction(
model_name="all-MiniLM-L6-v2"
)
)
self.groq_client = groq_client
self.model_name = "llama3-8b-8192"
def verify_claim(self, claim, confidence_threshold=0.5):
# Vector search returns full verified statements with distances
results = self.collection.query(
query_texts=[claim],
n_results=3,
include=["documents", "metadatas", "distances"]
)
# Pair documents with their distances and sort by similarity (ascending distance)
zipped_results = sorted(
zip(results['documents'][0], results['metadatas'][0], results['distances'][0]),
key=lambda x: x[2] # Sort by distance (ascending = most similar first)
)
# Format evidence with similarity scores (full sentences, not fragments)
evidence = []
for doc, meta, distance in zipped_results:
source = meta["source"] if meta and "source" in meta else "Unknown source"
# Convert distance to similarity score (higher = more similar)
similarity_score = 1 - (distance / 2) # Assuming cosine distance in [0,2]
evidence.append(
f'"{doc}" (Source: {source}, Similarity: {similarity_score:.2f})'
)
# Calculate overall confidence
avg_distance = sum(d for _, _, d in zipped_results) / len(zipped_results)
confidence = 1 - (avg_distance / 2) # Normalize to 0-1 range
# Threshold check
if confidence < confidence_threshold:
return {
"verdict": "Unverifiable",
"confidence": confidence,
"evidence": [e.split(" (Source:")[0] for e in evidence], # Cleaned evidence
"reasoning": "Claim is too vague or lacks sufficient evidence"
}
# LLM verification with distance-aware prompt
evidence_str = "\n".join([f"- {e}" for e in evidence])
prompt = f""" You are a powerful fact checker. Analyze the claim below against the provided verified information.
Relying on the similarity scores, also carefully check whether all factual details in the claim (such as dates, names, locations, and events) exactly match the evidence.
If there is any factual mismatch (for example, the date in the claim is different from the evidence), classify the claim as False. Any factual mismatch, even if the overall context is similar, should lead to a False classification.
If the evidence is too vague or lacks strong matches, classify as Unverifiable.
If evidence directly contradicts the claim, classify as False.
Any discrepancy in factual details, even if the overall context is similar, should lead to a False classification.
If the evidence fully supports the claim with all factual details matching, classify as True.
Claim:
{claim}
Evidence (with similarity scores):
{evidence_str}
Guidelines:
1. Give more weight to evidence with higher similarity scores, but do not ignore factual mismatches.
2. Pay close attention to details such as dates, names, locations, and events.
3. If the claim and evidence differ on any factual point, do not classify as True.
4. Respond only in JSON format without any additional text.
5. In the "evidence" array, include only full evidence statements as strings, without any extra comments or explanations.
6. Put all explanations or comparisons in the "reasoning" field.
Respond in JSON format:
{{
"verdict": "Verdict",
"evidence": [List of relevant facts from provided evidence],
"reasoning": "Explanation of the verdict based on evidence and factual details"
}}
"""
completion = self.groq_client.chat.completions.create(
model=self.model_name,
messages=[{"role": "user", "content": prompt}],
temperature=0.1,
max_tokens=400
)
# Process response
response_content = completion.choices[0].message.content
print(f"Response from Groq: {response_content}")
# Use the robust JSON extractor
parsed = robust_json_extractor(response_content)
print(f"Parsed JSON: {parsed}")
if "error" in parsed:
return {
"error": parsed["error"],
"confidence": confidence,
"raw_response": parsed.get("raw", response_content)
}
else:
# Validate required fields
required_keys = ["verdict", "evidence", "reasoning"]
if all(key in parsed for key in required_keys):
return {
"verdict": parsed["verdict"],
"confidence": confidence,
"evidence": [e.split(" (Source:")[0] for e in evidence],
"reasoning": parsed["reasoning"]
}
else:
return {
"error": f"Missing required keys: {[k for k in required_keys if k not in parsed]}",
"confidence": confidence,
"raw_response": response_content
} |