Update app.py

app.py

@@ -2,6 +2,9 @@ import streamlit as st
 import pandas as pd
 import numpy as np
 import random
+import os
+from datetime import datetime
+import ast
 
 class RoFTGame:
     def __init__(self, dataset_path):
@@ -15,13 +18,29 @@ class RoFTGame:
         self.current_sentences = None
         self.true_boundary_index = None
         self.current_guess_index = None
+        
+        # Predefined reasons from the dataset description
+        self.predefined_reasons = [
+            "grammar",
+            "repetition", 
+            "irrelevant", 
+            "contradicts_sentence", 
+            "contradicts_knowledge", 
+            "common_sense", 
+            "coreference", 
+            "generic"
+        ]
 
     def load_random_sample(self):
         """
         Load a random sample from the dataset
         """
-        # Filter for samples with valid generations
-        valid_samples = self.df[self.df['gen_body'].notna()]
+        # Filter for samples with valid generations and reasons
+        valid_samples = self.df[
+            (self.df['gen_body'].notna()) & 
+            (self.df['reason'].notna()) & 
+            (self.df['reason'] != '[]')
+        ]
         
         # Select a random sample
         self.current_sample = valid_samples.sample(n=1).iloc[0]
@@ -37,6 +56,12 @@ class RoFTGame:
         # Store true boundary
         self.true_boundary_index = self.current_sample['true_boundary_index']
         
+        # Parse reasons from the dataset
+        try:
+            self.current_reasons = ast.literal_eval(self.current_sample['reason'])
+        except:
+            self.current_reasons = []
+        
         # Reset current guess
         self.current_guess_index = None
 
@@ -57,6 +82,64 @@ class RoFTGame:
         else:
             return 0
 
+    def validate_reason(self, user_reason):
+        """
+        Validate user's reason against dataset reasons
+        
+        :param user_reason: Reason provided by user
+        :return: Tuple of (is_valid, matching_reasons)
+        """
+        # Convert user reason to lowercase for matching
+        user_reason_lower = user_reason.lower()
+        
+        # Check against predefined reasons and current sample's reasons
+        matching_reasons = []
+        
+        # Check predefined reasons
+        for reason in self.predefined_reasons:
+            if reason.lower() in user_reason_lower:
+                matching_reasons.append(reason)
+        
+        # Check original sample's reasons
+        for orig_reason in self.current_reasons:
+            if orig_reason.lower() in user_reason_lower:
+                matching_reasons.append(orig_reason)
+        
+        return len(matching_reasons) > 0, matching_reasons
+
+    def save_annotation(self, guess_index, reason, reason_validity):
+        """
+        Save annotation to a text file
+        
+        :param guess_index: Index of the guessed boundary
+        :param reason: Reason for the guess
+        :param reason_validity: Validity of the reason
+        """
+        # Ensure logs directory exists
+        os.makedirs('logs', exist_ok=True)
+        
+        # Generate unique filename with timestamp
+        timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+        filename = f'logs/annotation_{timestamp}.txt'
+        
+        # Prepare annotation details
+        annotation_details = [
+            f"Timestamp: {timestamp}",
+            f"Model: {self.current_sample['model']}",
+            f"Dataset: {self.current_sample['dataset']}",
+            f"Guess Index: {guess_index + 1}",
+            f"True Boundary Index: {self.true_boundary_index + 1}",
+            f"Original Dataset Reasons: {self.current_reasons}",
+            f"User Reason: {reason}",
+            f"Reason Validity: {reason_validity[0]}",
+            f"Matching Reasons: {reason_validity[1]}",
+            "\nFull Text:\n" + "\n".join(f"{i+1}. {sent}" for i, sent in enumerate(self.current_sentences))
+        ]
+        
+        # Write to file
+        with open(filename, 'w') as f:
+            f.write("\n".join(annotation_details))
+
 def main():
     st.title("Real or Fake Text (RoFT) Game")
     
@@ -80,29 +163,60 @@ def main():
     # Boundary selection
     guess = st.radio(
         "Where do you think the AI-generated text begins?", 
-        options=[f"Sentence {i+1}" for i in range(len(st.session_state.game.current_sentences))]
+        options=[str(i+1) for i in range(len(st.session_state.game.current_sentences))]
     )
     
+    # Reason input with predefined options
+    st.subheader("Reason Selection")
+    reason_options = st.session_state.game.predefined_reasons
+    selected_predefined_reasons = st.multiselect(
+        "Select predefined reasons (you can also add custom text)",
+        options=reason_options
+    )
+    
+    # Custom reason input
+    custom_reason = st.text_area("Additional explanation (optional)")
+    
+    # Combine predefined and custom reasons
+    full_reason = " ".join(selected_predefined_reasons)
+    if custom_reason:
+        full_reason += f" {custom_reason}"
+    
     # Guess submission
     if st.button("Submit Guess"):
-        # Convert guess to index
-        guess_index = int(guess.split()[-1]) - 1
-        
-        # Check guess and update points
-        points_earned = st.session_state.game.check_guess(guess_index)
-        st.session_state.total_points += points_earned
-        st.session_state.rounds_played += 1
-        
-        # Show results
-        st.subheader("Results")
-        st.write(f"Your Guess: {guess}")
-        st.write(f"Actual Boundary: Sentence {st.session_state.game.true_boundary_index + 1}")
-        st.write(f"Points Earned: {points_earned}")
-        
-        # Option to continue
-        if st.button("Next Round"):
-            st.session_state.game.load_random_sample()
-            st.experimental_rerun()
+        if not full_reason.strip():
+            st.warning("Please provide a reason for your guess.")
+        else:
+            # Convert guess to index (subtract 1 for 0-based indexing)
+            guess_index = int(guess) - 1
+            
+            # Check guess and update points
+            points_earned = st.session_state.game.check_guess(guess_index)
+            st.session_state.total_points += points_earned
+            st.session_state.rounds_played += 1
+            
+            # Validate reason
+            reason_validity = st.session_state.game.validate_reason(full_reason)
+            
+            # Save annotation
+            st.session_state.game.save_annotation(guess_index, full_reason, reason_validity)
+            
+            # Show results
+            st.subheader("Results")
+            st.write(f"Your Guess: Sentence {guess}")
+            st.write(f"Actual Boundary: Sentence {st.session_state.game.true_boundary_index + 1}")
+            st.write(f"Points Earned: {points_earned}")
+            
+            # Display reason validation
+            st.write("Reason Validation:")
+            st.write(f"Valid Reason: {reason_validity[0]}")
+            if reason_validity[1]:
+                st.write("Matching Reasons:", ", ".join(reason_validity[1]))
+            
+            # Option to continue
+            if st.button("Next Round"):
+                st.session_state.game.load_random_sample()
+                st.experimental_rerun()
 
     # Optional: Show metadata for current sample
     if st.checkbox("Show Sample Metadata"):
@@ -111,6 +225,7 @@ def main():
         st.write(f"Model: {sample['model']}")
         st.write(f"Dataset: {sample['dataset']}")
         st.write(f"Sampling Strategy (p): {sample['dec_strat_value']}")
+        st.write(f"Original Reasons: {st.session_state.game.current_reasons}")
 
 if __name__ == "__main__":
     main()