Synced repo using 'sync_with_huggingface' Github Action

app.py

@@ -154,34 +154,81 @@ button[kind="primary"] {
                 st.session_state.last_claim = claim
                 st.session_state.feedback_submitted = False  # Reset feedback state for new claim
 
-        # Display results from session state
-        if st.session_state.result:
-            result = st.session_state.result
-            if "error" in result:
-                st.error(f"Error: {result['error']}")
-                if "raw_response" in result:
-                    with st.expander("Show raw LLM response"):
-                        st.code(result["raw_response"])
-            else:
-                # Display verdict
-                verdict_color = {
-                    "True": "green",
-                    "False": "red",
-                    "Unverifiable": "orange"
-                }.get(result["verdict"], "gray")
-                st.markdown(f"**Verdict:** :{verdict_color}[{result['verdict']}]")
-                
-                # Display confidence score
-                st.metric("Confidence Score", f"{result.get('confidence', 0):.2f}")
-                
-                # Display evidence
-                with st.expander("View Supporting Evidence"):
-                    for idx, evidence in enumerate(result.get("evidence", []), 1):
-                        st.markdown(f"{idx}. {evidence}")
-                
-                # Display reasoning
-                st.markdown("**Analysis:**")
-                st.write(result.get("reasoning", "No reasoning provided"))
+            # Display results from session state
+            if st.session_state.result:
+                result = st.session_state.result
+
+                # Show entity verification results
+                st.subheader("Entity Verification Results")
+                entities = result.get("entities", [])
+                if entities:
+                    for idx, entity_result in enumerate(entities, 1):
+                        st.markdown(f"### Entity {idx}: {entity_result.get('entity', '')} ({entity_result.get('type', '')})")
+
+                        if "error" in entity_result:
+                            st.error(f"Error: {entity_result['error']}")
+                            if "raw_response" in entity_result:
+                                with st.expander("Show raw LLM response"):
+                                    st.code(entity_result["raw_response"])
+                            continue
+
+                        verdict_color = {
+                            "Valid": "green",
+                            "Invalid": "red",
+                            "Unverified": "orange"
+                        }.get(entity_result.get("verdict", ""), "gray")
+                        st.markdown(f"**Verdict:** :{verdict_color}[{entity_result.get('verdict', 'Unknown')}]")
+
+                        # Confidence
+                        st.metric("Confidence Score", f"{entity_result.get('confidence', 0):.2f}")
+
+                        # Evidence
+                        with st.expander("View Supporting Evidence"):
+                            for i, evidence in enumerate(entity_result.get("evidence", []), 1):
+                                st.markdown(f"{i}. {evidence}")
+
+                        # Reasoning
+                        st.markdown("**Analysis:**")
+                        st.write(entity_result.get("reasoning", "No reasoning provided"))
+                else:
+                    st.write("No entities detected or verified.")
+
+                # Show claim verification results
+                st.subheader("Detected Claims and Verification Results")
+                claims = result.get("claims", [])
+                if not claims:
+                    st.info("No check-worthy claims detected in the input.")
+                else:
+                    for idx, claim_result in enumerate(claims, 1):
+                        st.markdown(f"### Claim {idx}")
+                        st.markdown(f"> {claim_result.get('claim', '')}")
+
+                        if "error" in claim_result:
+                            st.error(f"Error: {claim_result['error']}")
+                            if "raw_response" in claim_result:
+                                with st.expander("Show raw LLM response"):
+                                    st.code(claim_result["raw_response"])
+                            continue
+
+                        verdict_color = {
+                            "True": "green",
+                            "False": "red",
+                            "Unverifiable": "orange"
+                        }.get(claim_result.get("verdict", ""), "gray")
+                        st.markdown(f"**Verdict:** :{verdict_color}[{claim_result.get('verdict', 'Unknown')}]")
+
+                        # Confidence
+                        st.metric("Confidence Score", f"{claim_result.get('confidence', 0):.2f}")
+
+                        # Evidence
+                        with st.expander("View Supporting Evidence"):
+                            for i, evidence in enumerate(claim_result.get("evidence", []), 1):
+                                st.markdown(f"{i}. {evidence}")
+
+                        # Reasoning
+                        st.markdown("**Analysis:**")
+                        st.write(claim_result.get("reasoning", "No reasoning provided"))
+
 
             # Feedback system
             feedback_key = f"feedback_radio_{st.session_state.last_claim}"

fact_checker.py

@@ -6,6 +6,9 @@ import re
 from openai import OpenAI
 import re
 import json
+import spacy
+from transformers import T5ForConditionalGeneration, T5Tokenizer
+from transformers import pipeline  
 
 def robust_json_extractor(response_content):
     # Preprocess: Remove markdown code blocks and extra whitespace
@@ -56,49 +59,55 @@ class FactChecker:
         )
         self.groq_client = groq_client
         self.model_name = "llama3-8b-8192"
+        self.ner = spacy.load("en_core_web_sm")
+        
+
+        self.claim_tokenizer = T5Tokenizer.from_pretrained("Babelscape/t5-base-summarization-claim-extractor")
+        self.claim_model = T5ForConditionalGeneration.from_pretrained("Babelscape/t5-base-summarization-claim-extractor")
+
+    def extract_entities(self, text):
+        doc = self.ner(text)
+        return [(ent.text, ent.label_) for ent in doc.ents]
+
+    def extract_claims(self, text, threshold=0.5):
+        tok_input = self.claim_tokenizer.batch_encode_plus([text], return_tensors="pt", padding=True)
+        outputs = self.claim_model.generate(**tok_input)
+        claims = self.claim_tokenizer.batch_decode(outputs, skip_special_tokens=True)
+        claims = [claim.strip() for claim in claims if len(claim.strip()) > 0]
+        return claims
+
 
-    def verify_claim(self, claim, confidence_threshold=0.5):
-    # Vector search returns full verified statements with distances
+    def verify_single_claim(self, claim, confidence_threshold=0.5):
         results = self.collection.query(
             query_texts=[claim],
             n_results=3,
             include=["documents", "metadatas", "distances"]
         )
-        
-        # Pair documents with their distances and sort by similarity (ascending distance)
         zipped_results = sorted(
             zip(results['documents'][0], results['metadatas'][0], results['distances'][0]),
-            key=lambda x: x[2]  # Sort by distance (ascending = most similar first)
+            key=lambda x: x[2]
         )
-        
-        # Format evidence with similarity scores (full sentences, not fragments)
         evidence = []
         for doc, meta, distance in zipped_results:
             source = meta["source"] if meta and "source" in meta else "Unknown source"
-            # Convert distance to similarity score (higher = more similar)
             similarity_score = 1 - (distance / 2)  # Assuming cosine distance in [0,2]
             evidence.append(
                 f'"{doc}" (Source: {source}, Similarity: {similarity_score:.2f})'
             )
-
-
-        # Calculate overall confidence
         avg_distance = sum(d for _, _, d in zipped_results) / len(zipped_results)
         confidence = 1 - (avg_distance / 2)  # Normalize to 0-1 range
 
-        # Threshold check
         if confidence < confidence_threshold:
             return {
                 "verdict": "Unverifiable",
                 "confidence": confidence,
-                "evidence": [e.split(" (Source:")[0] for e in evidence],  # Cleaned evidence
+                "evidence": [e.split(" (Source:")[0] for e in evidence],
                 "reasoning": "Claim is too vague or lacks sufficient evidence"
             }
 
-        # LLM verification with distance-aware prompt
         evidence_str = "\n".join([f"- {e}" for e in evidence])
-        prompt = f""" You are a powerful fact checker. Analyze the claim below against the provided verified information. 
-Relying on the similarity scores, also carefully check whether all factual details in the claim (such as dates, names, locations, and events) exactly match atleast one of the evidence. If from first evidence, evidence is not sufficient, use the next evidence to verify the claim. 
+        prompt = f"""You are a powerful fact checker. Analyze the claim below against the provided verified information. 
+Relying on the similarity scores, also carefully check whether all factual details in the claim (such as dates, names, locations, and events) exactly match the evidence. 
 If there is any factual mismatch (for example, the date in the claim is different from the evidence), classify the claim as False. Any factual mismatch, even if the overall context is similar, should lead to a False classification.
 If the evidence is too vague or lacks strong matches, classify as Unverifiable.
 If evidence directly contradicts the claim, classify as False.
@@ -113,7 +122,6 @@ Evidence (with similarity scores):
 
 Guidelines:
 1. Give more weight to evidence with higher similarity scores, but do not ignore factual mismatches.
-2. If any one piece of evidence independently supports the claim, without factual mismatches, classify as True.
 2. Pay close attention to details such as dates, names, locations, and events.
 3. If the claim and evidence differ on any factual point, do not classify as True.
 4. Respond only in JSON format without any additional text.
@@ -127,23 +135,14 @@ Respond in JSON format:
     "reasoning": "Explanation of the verdict based on evidence and factual details"
 }}
 """
-
-        
         completion = self.groq_client.chat.completions.create(
             model=self.model_name,
             messages=[{"role": "user", "content": prompt}],
             temperature=0.1,
             max_tokens=400
         )
-        
-        # Process response
         response_content = completion.choices[0].message.content
-        print(f"Response from Groq: {response_content}")
-
-        # Use the robust JSON extractor
         parsed = robust_json_extractor(response_content)
-        print(f"Parsed JSON: {parsed}")
-
         if "error" in parsed:
             return {
                 "error": parsed["error"],
@@ -151,7 +150,6 @@ Respond in JSON format:
                 "raw_response": parsed.get("raw", response_content)
             }
         else:
-            # Validate required fields
             required_keys = ["verdict", "evidence", "reasoning"]
             if all(key in parsed for key in required_keys):
                 return {
@@ -165,4 +163,113 @@ Respond in JSON format:
                     "error": f"Missing required keys: {[k for k in required_keys if k not in parsed]}",
                     "confidence": confidence,
                     "raw_response": response_content
-                }
+                }
+
+    def verify_single_entity(self, entity_text, confidence_threshold=0.5):
+        """Verify a single named entity against the fact database"""
+        # Vector similarity search
+        results = self.collection.query(
+            query_texts=[entity_text],
+            n_results=3,
+            include=["documents", "metadatas", "distances"]
+        )
+        
+        # Process evidence with similarity normalization
+        evidence = []
+        total_distance = 0
+        for doc, meta, distance in zip(results['documents'][0], 
+                                    results['metadatas'][0], 
+                                    results['distances'][0]):
+            similarity = 1 - (distance / 2)  # Convert cosine distance to similarity
+            evidence.append({
+                "text": doc,
+                "source": meta.get("source", "Unknown"),
+                "similarity": similarity
+            })
+            total_distance += distance
+        
+        avg_similarity = 1 - (total_distance / len(results['distances'][0]) / 2)
+        
+        # Prepare LLM verification prompt
+        evidence_str = "\n".join([
+            f"- {e['text']} (Similarity: {e['similarity']:.2f})" 
+            for e in evidence
+        ])
+        
+        prompt = f"""**Entity Verification Task**
+    Entity: "{entity_text}"
+
+    **Verified Evidence:**
+    {evidence_str}
+
+    **Instructions:**
+    1. Verify if this entity exists in official records
+    2. Check for exact matches of names/titles
+    3. Confirm associated details (locations, dates, roles)
+    4. Return JSON with: verdict (True/False/Unverified), confidence (0-1), reasoning
+
+    **JSON Response:"""
+        
+        try:
+            response = self.groq_client.chat.completions.create(
+                model=self.model_name,
+                messages=[{"role": "user", "content": prompt}],
+                temperature=0.2,
+                response_format={"type": "json_object"}
+            )
+            
+            result = json.loads(response.choices[0].message.content)
+            return {
+                "verdict": result.get("verdict", "Unverified"),
+                "confidence": min(max(result.get("confidence", avg_similarity), 0), 1),
+                "evidence": [e["text"] for e in evidence],
+                "reasoning": result.get("reasoning", "No reasoning provided")
+            }
+            
+        except Exception as e:
+            return {
+                "verdict": "Error",
+                "confidence": 0,
+                "evidence": [],
+                "reasoning": f"Verification failed: {str(e)}"
+            }
+
+    def verify_claim(self, text, confidence_threshold=0.5):
+        """
+        Main method: takes input text, extracts entities and claims, 
+        verifies each, and returns JSON results
+        """
+        # Extract entities and claims
+        entities = self.extract_entities(text)
+        claims = self.extract_claims(text)
+        
+        # Verify claims
+        claim_results = []
+        for claim in claims:
+            verification = self.verify_single_claim(claim, confidence_threshold)
+            claim_results.append({
+                "claim": claim,
+                "verdict": verification.get("verdict", "Error"),
+                "confidence": verification.get("confidence", 0),
+                "evidence": verification.get("evidence", []),
+                "reasoning": verification.get("reasoning", "Analysis failed")
+            })
+        
+        # Verify entities
+        entity_results = []
+        for entity_text, entity_label in entities:
+            verification = self.verify_single_entity(entity_text, confidence_threshold)
+            entity_results.append({
+                "entity": entity_text,
+                "type": entity_label,
+                "verdict": verification.get("verdict", "Error"),
+                "confidence": verification.get("confidence", 0),
+                "evidence": verification.get("evidence", []),
+                "reasoning": verification.get("reasoning", "Analysis failed")
+            })
+        
+        return {
+            "entities": entity_results,
+            "claims": claim_results
+        }
+