app.py

import re
import numpy as np
import json
from sentence_transformers import SentenceTransformer
from transformers import AutoTokenizer
from langchain_google_genai import ChatGoogleGenerativeAI
import os
import gradio as gr
import time


tokenizer = AutoTokenizer.from_pretrained("answerdotai/ModernBERT-base")
sentence_model = SentenceTransformer('all-MiniLM-L6-v2')

def clean_text(text):
    text = re.sub(r'\[speaker_\d+\]', '', text)
    text = re.sub(r'\s+', ' ', text).strip()
    return text

def split_text_by_tokens(text, max_tokens=8000):
    text = clean_text(text)
    tokens = tokenizer.encode(text)
    
    if len(tokens) <= max_tokens:
        return [text]
    
    split_point = len(tokens) // 2
    
    sentences = re.split(r'(?<=[.!?])\s+', text)
    
    first_half = []
    second_half = []
    
    current_tokens = 0
    for sentence in sentences:
        sentence_tokens = len(tokenizer.encode(sentence))
        
        if current_tokens + sentence_tokens <= split_point:
            first_half.append(sentence)
            current_tokens += sentence_tokens
        else:
            second_half.append(sentence)
    
    return [" ".join(first_half), " ".join(second_half)]

def analyze_segment_with_gemini(segment_text):
    llm = ChatGoogleGenerativeAI(
        model="gemini-1.5-flash",
        temperature=0.7,
        max_tokens=None,
        timeout=None,
        max_retries=3
    )
    
    prompt = f"""
        Analyze the following text and identify distinct segments within it and do text segmentation:
        1. Segments should be STRICTLY max=15
        2. For each segment/topic you identify:
           - Provide a SPECIFIC and UNIQUE topic name (3-5 words) that clearly differentiates it from other segments
           - List 3-5 key concepts discussed in that segment (be precise and avoid repetition between segments)
           - Write a brief summary of that segment (3-5 sentences)
           - Create 5 high-quality, meaningful quiz questions based DIRECTLY on the content in that segment only
           - Questions and answers should be only from the content of the segment

        For each quiz question:
        - Create one correct answer that comes DIRECTLY from the text
        - Create two plausible but incorrect answers
        - IMPORTANT: Ensure all answer options have similar length (± 3 words)
        - Ensure the correct answer is clearly indicated with a ✓ symbol
       - Questions should **require actual understanding**, NOT just basic fact recall.
        - Questions Are **non-trivial**, encourage deeper thinking, and **avoid surface-level facts**.
        - Are **directly based on the segment's content** (not inferred from the summary).
        - Do **not include questions about document structure** (e.g., title, number of paragraphs).
        - Do **not generate overly generic or obvious questions** (e.g., "What is mentioned in the text?").
        - Focus on **core ideas, logical reasoning, and conceptual understanding**.

    ADDITIONAL REQUIREMENT:
    - **First, detect the language of the original text.**
    - **Generate ALL output (topic names, key concepts, summaries, and quizzes) in the same language as the original text.**
    - If the text is in Russian, generate all responses in Russian.
    - If the text is in another language, generate responses in that original language.
    

        Text:
        {segment_text}

        Format your response as JSON with the following structure:
        {{
            "segments": [
                {{
                    "topic_name": "Unique and Specific Topic Name",
                    "key_concepts": ["concept1", "concept2", "concept3"],
                    "summary": "Brief summary of this segment.",
                    "quiz_questions": [
                        {{
                            "question": "Question text?",
                            "options": [
                                {{
                                    "text": "Option A",
                                    "correct": false
                                }},
                                {{
                                    "text": "Option B",
                                    "correct": true
                                }},
                                {{
                                    "text": "Option C",
                                    "correct": false
                                }}
                            ]
                        }}
                    ]
                }}
            ]
        }}

        IMPORTANT: Each segment must have a DISTINCT topic name that clearly differentiates it from others.
    - **Do NOT repeat** key concepts across multiple segments unless absolutely necessary.
    - **Ensure the quiz questions challenge the reader** and **are not easily guessable**.

    """
    
    response = llm.invoke(prompt)
    response_text = response.content
    
    try:
        json_match = re.search(r'\{[\s\S]*\}', response_text)
        if json_match:
            return json.loads(json_match.group(0))
        else:
            return json.loads(response_text)
    except json.JSONDecodeError:
        return {
            "segments": [
                {
                    "topic_name": "JSON Parsing Error",
                    "key_concepts": ["Error in response format"],
                    "summary": "Could not parse the API response.",
                    "quiz_questions": []
                }
            ]
        }

def process_document_with_quiz(text):
    start_time = time.time()
    
    token_count = len(tokenizer.encode(text))
    print(f"[LOG] Total document tokens: {token_count}")
    
    if token_count > 8000:
        print(f"[LOG] Document exceeds 8000 tokens. Splitting into parts.")
        parts = split_text_by_tokens(text)
        print(f"[LOG] Document split into {len(parts)} parts")
        
        for i, part in enumerate(parts):
            part_tokens = len(tokenizer.encode(part))
            print(f"[LOG] Part {i+1} contains {part_tokens} tokens")
    else:
        print(f"[LOG] Document under 8000 tokens. Processing as a single part.")
        parts = [text]
    
    all_segments = []
    segment_counter = 1
    
    for i, part in enumerate(parts):
        part_start_time = time.time()
        print(f"[LOG] Processing part {i+1}...")
        
        analysis = analyze_segment_with_gemini(part)
        
        if "segments" in analysis:
            print(f"[LOG] Found {len(analysis['segments'])} segments in part {i+1}")
            
            for segment in analysis["segments"]:
                segment["segment_number"] = segment_counter
                all_segments.append(segment)
                print(f"[LOG] Segment {segment_counter}: {segment['topic_name']}")
                segment_counter += 1
        else:
            # Fallback if response format is unexpected
            print(f"[LOG] Error: Unexpected format in part {i+1} analysis")
            fallback_segment = {
                "topic_name": f"Segment {segment_counter} Analysis",
                "key_concepts": ["Format error in analysis"],
                "summary": "Could not properly segment this part of the text.",
                "quiz_questions": [],
                "segment_number": segment_counter
            }
            all_segments.append(fallback_segment)
            print(f"[LOG] Added fallback segment {segment_counter}")
            segment_counter += 1
            
        part_time = time.time() - part_start_time
        print(f"[LOG] Part {i+1} processed in {part_time:.2f} seconds")
    
    total_time = time.time() - start_time
    print(f"[LOG] Total processing time: {total_time:.2f} seconds")
    print(f"[LOG] Generated {len(all_segments)} segments total")
    
    return all_segments


def format_quiz_for_display(results):
    output = []
    
    for segment in results:
        topic = segment["topic_name"]
        segment_num = segment["segment_number"]
        
        output.append(f"\n\n{'='*40}")
        output.append(f"SEGMENT {segment_num}: {topic}")
        output.append(f"{'='*40}\n")
        
        output.append("KEY CONCEPTS:")
        for concept in segment["key_concepts"]:
            output.append(f"• {concept}")
        
        output.append("\nSUMMARY:")
        output.append(segment["summary"])
        
        output.append("\nQUIZ QUESTIONS:")
        for i, q in enumerate(segment["quiz_questions"]):
            output.append(f"\n{i+1}. {q['question']}")
            
            for j, option in enumerate(q['options']):
                letter = chr(97 + j).upper() 
                correct_marker = " ✓" if option["correct"] else ""
                output.append(f"   {letter}. {option['text']}{correct_marker}")
    
    return "\n".join(output)

def save_results_as_json(results, filename="analysis_results.json"):
    with open(filename, "w", encoding="utf-8") as f:
        json.dump(results, f, indent=2, ensure_ascii=False)
    return filename

def save_results_as_txt(formatted_text, filename="analysis_results.txt"):
    with open(filename, "w", encoding="utf-8") as f:
        f.write(formatted_text)
    return filename


def analyze_document(document_text, api_key):
    print(f"[LOG] Starting document analysis...")
    overall_start_time = time.time()
    
    os.environ["GOOGLE_API_KEY"] = api_key
    try:
        results = process_document_with_quiz(document_text)
        formatted_output = format_quiz_for_display(results)
        
        json_path = "analysis_results.json"
        txt_path = "analysis_results.txt"
        
        with open(json_path, "w", encoding="utf-8") as f:
            json.dump(results, f, indent=2, ensure_ascii=False)
        
        with open(txt_path, "w", encoding="utf-8") as f:
            f.write(formatted_output)
        
        overall_time = time.time() - overall_start_time
        print(f"[LOG] Document analysis completed in {overall_time:.2f} seconds")
        
        topics_summary = "DOCUMENT ANALYSIS SUMMARY:\n"
        topics_summary += f"Total segments: {len(results)}\n"
        topics_summary += f"Processing time: {overall_time:.2f} seconds\n\n"
        topics_summary += "SEGMENTS:\n"
        
        for segment in results:
            topics_summary += f"- Segment {segment['segment_number']}: {segment['topic_name']}\n"
        
        formatted_output = topics_summary + "\n" + formatted_output
        
        return formatted_output, json_path, txt_path
    except Exception as e:
        error_msg = f"Error processing document: {str(e)}"
        print(f"[LOG] ERROR: {error_msg}")
        return error_msg, None, None

with gr.Blocks(title="Quiz Generator") as app:
    gr.Markdown("# Quiz Generator")
    
    with gr.Row():
        with gr.Column():
            input_text = gr.Textbox(
                label="Input Document Text",
                placeholder="Paste your document text here...",
                lines=10
            )
            
            api_key = gr.Textbox(
                label="Gemini API Key",
                placeholder="Enter your Gemini API key",
                type="password"
            )
            
            analyze_btn = gr.Button("Analyze Document")
            
        with gr.Column():
            output_results = gr.Textbox(
                label="Analysis Results",
                lines=20
            )
            json_file_output = gr.File(label="Download JSON")
            txt_file_output = gr.File(label="Download TXT")
    
    analyze_btn.click(
        fn=analyze_document,
        inputs=[input_text, api_key],
        outputs=[output_results, json_file_output, txt_file_output]
    )

if __name__ == "__main__":
    app.launch()