Update app.py

app.py

@@ -1,21 +1,30 @@
-import gradio as gr
-import groq
 import os
 import tempfile
 import uuid
-from dotenv import load_dotenv
-from langchain.text_splitter import RecursiveCharacterTextSplitter
-from langchain.vectorstores import FAISS
-from langchain.embeddings import HuggingFaceEmbeddings
-import fitz  # PyMuPDF
 import base64
-from PIL import Image
 import io
-import requests
 import json
 import re
 from datetime import datetime, timedelta
 
+# Third-party imports
+import gradio as gr
+import groq
+import numpy as np
+import pandas as pd
+import openpyxl
+import requests
+import fitz  # PyMuPDF
+from PIL import Image
+from dotenv import load_dotenv
+from transformers import AutoProcessor, AutoModelForVision2Seq
+import torch
+
+# LangChain imports
+from langchain_community.embeddings import HuggingFaceEmbeddings
+from langchain_community.vectorstores import FAISS
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+
 # Load environment variables
 load_dotenv()
 client = groq.Client(api_key=os.getenv("GROQ_TECH_API_KEY"))
@@ -29,6 +38,19 @@ if not os.path.exists(FAISS_INDEX_DIR):
 # Dictionary to store user-specific vectorstores
 user_vectorstores = {}
 
+# Load SmolDocling model for image analysis
+def load_docling_model():
+    try:
+        processor = AutoProcessor.from_pretrained("ds4sd/SmolDocling-256M-preview")
+        model = AutoModelForVision2Seq.from_pretrained("ds4sd/SmolDocling-256M-preview")
+        return processor, model
+    except Exception as e:
+        print(f"Error loading SmolDocling model: {e}")
+        return None, None
+
+# Initialize SmolDocling model
+docling_processor, docling_model = load_docling_model()
+
 # Custom CSS for Tech theme
 custom_css = """
 :root {
@@ -70,6 +92,90 @@ body { background-color: var(--light-background); font-family: 'Google Sans', 'R
 .qa-body { color: var(--dark-text); font-size: 0.95rem; margin-bottom: 10px; }
 .qa-meta { display: flex; justify-content: space-between; color: #5F6368; font-size: 0.85rem; }
 .tag { background-color: #E8F0FE; color: var(--primary-color); padding: 4px 8px; border-radius: 4px; font-size: 0.8rem; margin-right: 5px; display: inline-block; }
+.toggle-container { display: flex; align-items: center; margin-bottom: 15px; }
+.toggle-label { margin-right: 10px; font-weight: 500; }
+.search-toggle { margin-left: 5px; }
+.voice-btn { background-color: var(--primary-color) !important; border-radius: 50% !important; width: 44px !important; height: 44px !important; display: flex !important; align-items: center !important; justify-content: center !important; color: var(--white) !important; box-shadow: 0 2px 5px rgba(0,0,0,0.2) !important; }
+.speak-btn { background-color: var(--secondary-color) !important; border-radius: 24px !important; color: var(--white) !important; padding: 8px 16px !important; font-weight: 500 !important; margin-left: 10px !important; }
+.audio-controls { display: flex; align-items: center; margin-top: 10px; }
+/* Audio Visualization Elements */
+.audio-visualization {
+    display: flex;
+    align-items: center;
+    justify-content: center;
+    gap: 4px;
+    height: 40px;
+    padding: 10px;
+    background-color: rgba(0,0,0,0.05);
+    border-radius: 12px;
+    margin: 10px 0;
+}
+.audio-bar {
+    width: 3px;
+    background-color: var(--accent-color);
+    border-radius: 2px;
+    height: 5px;
+    transition: height 0.1s ease;
+}
+.audio-status {
+    font-size: 0.85rem;
+    color: var(--secondary-color);
+    text-align: center;
+    margin-top: 5px;
+    font-style: italic;
+}
+.recording-indicator {
+    width: 12px;
+    height: 12px;
+    border-radius: 50%;
+    background-color: #ff4b4b;
+    margin-right: 8px;
+    animation: blink 1s infinite;
+}
+.playing-indicator {
+    width: 12px;
+    height: 12px;
+    border-radius: 50%;
+    background-color: #4bff4b;
+    margin-right: 8px;
+    animation: pulse 1s infinite;
+}
+@keyframes blink {
+    0% { opacity: 1; }
+    50% { opacity: 0.4; }
+    100% { opacity: 1; }
+}
+@keyframes pulse {
+    0% { transform: scale(1); }
+    50% { transform: scale(1.2); }
+    100% { transform: scale(1); }
+}
+.file-upload-enhancement .file-preview {
+    max-height: 200px;
+    overflow: auto;
+    border: 1px solid var(--border-color);
+    border-radius: 8px;
+    padding: 10px;
+    margin-top: 10px;
+    background-color: rgba(0,0,0,0.02);
+}
+.excel-preview-table {
+    width: 100%;
+    border-collapse: collapse;
+    font-size: 0.85rem;
+}
+.excel-preview-table th, .excel-preview-table td {
+    border: 1px solid #ddd;
+    padding: 4px 8px;
+    text-align: left;
+}
+.excel-preview-table th {
+    background-color: var(--secondary-color);
+    color: white;
+}
+.excel-preview-table tr:nth-child(even) {
+    background-color: rgba(0,0,0,0.03);
+}
 """
 
 # Function to process PDF files
@@ -109,8 +215,146 @@ def process_pdf(pdf_file):
             os.unlink(pdf_path)
         return None, f"Error processing PDF: {str(e)}", {"page_images": [], "total_pages": 0, "total_words": 0}
 
+# New function to process Excel files
+def process_excel(excel_file):
+    if excel_file is None:
+        return None, "No file uploaded", {"data_preview": "", "total_sheets": 0, "total_rows": 0}
+    
+    try:
+        session_id = str(uuid.uuid4())
+        with tempfile.NamedTemporaryFile(suffix=".xlsx", delete=False) as temp_file:
+            temp_file.write(excel_file)
+            excel_path = temp_file.name
+        
+        # Read Excel file with pandas
+        excel_data = pd.ExcelFile(excel_path)
+        sheet_names = excel_data.sheet_names
+        all_texts = []
+        total_rows = 0
+        
+        # Process each sheet
+        for sheet in sheet_names:
+            df = pd.read_excel(excel_path, sheet_name=sheet)
+            total_rows += len(df)
+            
+            # Convert dataframe to text for vectorization
+            sheet_text = f"Sheet: {sheet}\n"
+            sheet_text += df.to_string(index=False)
+            all_texts.append(sheet_text)
+        
+        # Generate HTML preview of first sheet
+        first_df = pd.read_excel(excel_path, sheet_name=0)
+        preview_rows = min(10, len(first_df))
+        data_preview = first_df.head(preview_rows).to_html(classes="excel-preview-table", index=False)
+        
+        # Process for vectorstore
+        text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=200)
+        chunks = text_splitter.create_documents(all_texts)
+        vectorstore = FAISS.from_documents(chunks, embeddings)
+        index_path = os.path.join(FAISS_INDEX_DIR, session_id)
+        vectorstore.save_local(index_path)
+        user_vectorstores[session_id] = vectorstore
+
+        os.unlink(excel_path)
+        excel_state = {"data_preview": data_preview, "total_sheets": len(sheet_names), "total_rows": total_rows}
+        return session_id, f"✅ Successfully processed {len(chunks)} text chunks from Excel file", excel_state
+    except Exception as e:
+        if "excel_path" in locals() and os.path.exists(excel_path):
+            os.unlink(excel_path)
+        return None, f"Error processing Excel file: {str(e)}", {"data_preview": "", "total_sheets": 0, "total_rows": 0}
+
+# Function to analyze image using SmolDocling
+def analyze_image(image_file):
+    if image_file is None:
+        return "No image uploaded. Please upload an image to analyze."
+    
+    if docling_processor is None or docling_model is None:
+        return "SmolDocling model not loaded. Please check your installation."
+    
+    try:
+        # Process the image - image_file is a filepath string from Gradio
+        image = Image.open(image_file)
+        
+        # Use the SmolDocling model
+        inputs = docling_processor(images=image, return_tensors="pt")
+        with torch.no_grad():
+            outputs = docling_model.generate(
+                **inputs, 
+                max_new_tokens=512,
+                temperature=0.1,
+                do_sample=False
+            )
+        
+        # Decode the output
+        result = docling_processor.batch_decode(outputs, skip_special_tokens=True)[0]
+        
+        # Format the result for display with technical emphasis
+        analysis = f"## Technical Document Analysis Results\n\n{result}\n\n"
+        analysis += "### Technical Insights\n\n"
+        analysis += "* The analysis provides technical information extracted from the document image.\n"
+        analysis += "* Consider this information as a starting point for further technical investigation.\n"
+        analysis += "* For code snippets or technical specifications, verify accuracy before implementation.\n"
+        
+        return analysis
+    except Exception as e:
+        return f"Error analyzing image: {str(e)}"
+
+# Function to handle different file types
+def process_file(file_data, file_type):
+    if file_data is None:
+        return None, "No file uploaded", None
+    
+    if file_type == "pdf":
+        return process_pdf(file_data)
+    elif file_type == "excel":
+        return process_excel(file_data)
+    elif file_type == "image":
+        # For image files, we'll just use them directly for analysis
+        # But we'll return a session ID to maintain consistency
+        session_id = str(uuid.uuid4())
+        return session_id, "✅ Image file ready for analysis", None
+    else:
+        return None, "Unsupported file type", None
+
+# Function for speech-to-text conversion
+def speech_to_text():
+    try:
+        r = sr.Recognizer()
+        with sr.Microphone() as source:
+            r.adjust_for_ambient_noise(source)
+            audio = r.listen(source)
+            text = r.recognize_google(audio)
+            return text
+    except sr.UnknownValueError:
+        return "Could not understand audio. Please try again."
+    except sr.RequestError as e:
+        return f"Error with speech recognition service: {e}"
+    except Exception as e:
+        return f"Error converting speech to text: {str(e)}"
+
+# Function for text-to-speech conversion
+def text_to_speech(text, history):
+    if not text or not history:
+        return None
+    
+    try:
+        # Get the last bot response
+        last_response = history[-1][1]
+        
+        # Convert text to speech
+        tts = pyttsx3.init()
+        tts.setProperty('rate', 150)
+        tts.setProperty('volume', 0.9)
+        tts.save_to_file(last_response, "temp_output.mp3")
+        tts.runAndWait()
+        
+        return "temp_output.mp3"
+    except Exception as e:
+        print(f"Error in text-to-speech: {e}")
+        return None
+
 # Function to generate chatbot responses with Tech theme
-def generate_response(message, session_id, model_name, history):
+def generate_response(message, session_id, model_name, history, web_search_enabled=True):
     if not message:
         return history
     try:
@@ -121,8 +365,8 @@ def generate_response(message, session_id, model_name, history):
             if docs:
                 context = "\n\nRelevant information from uploaded PDF:\n" + "\n".join(f"- {doc.page_content}" for doc in docs)
         
-        # Check if it's a GitHub repo search
-        if re.match(r'^/github\s+.+', message, re.IGNORECASE):
+        # Check if it's a GitHub repo search and web search is enabled
+        if web_search_enabled and re.match(r'^/github\s+.+', message, re.IGNORECASE):
             query = re.sub(r'^/github\s+', '', message, flags=re.IGNORECASE)
             repo_results = search_github_repos(query)
             if repo_results:
@@ -139,8 +383,8 @@ def generate_response(message, session_id, model_name, history):
                 history.append((message, "No GitHub repositories found for your query."))
                 return history
         
-        # Check if it's a Stack Overflow search
-        if re.match(r'^/stack\s+.+', message, re.IGNORECASE):
+        # Check if it's a Stack Overflow search and web search is enabled
+        if web_search_enabled and re.match(r'^/stack\s+.+', message, re.IGNORECASE):
             query = re.sub(r'^/stack\s+', '', message, flags=re.IGNORECASE)
             qa_results = search_stackoverflow(query)
             if qa_results:
@@ -437,106 +681,155 @@ def perform_stack_search(query, tag, sort_by):
 with gr.Blocks(css=custom_css, theme=gr.themes.Soft()) as demo:
     current_session_id = gr.State(None)
     pdf_state = gr.State({"page_images": [], "total_pages": 0, "total_words": 0})
+    excel_state = gr.State({"data_preview": "", "total_sheets": 0, "total_rows": 0})
+    file_type = gr.State("none")
+    audio_status = gr.State("Ready")
+    
     gr.HTML("""
     <div class="header">
-        <div class="header-title">Tech-Vision</div>
-        <div class="header-subtitle">Analyze technical documents with Groq's LLM API.</div>
+        <div class="header-title">Tech-Vision Enhanced</div>
+        <div class="header-subtitle">Analyze technical documents, spreadsheets, and images with AI</div>
     </div>
     """)
     with gr.Row(elem_classes="container"):
         with gr.Column(scale=1, min_width=300):
-            pdf_file = gr.File(label="Upload PDF Document", file_types=[".pdf"], type="binary")
-            upload_button = gr.Button("Process PDF", variant="primary")
-            pdf_status = gr.Markdown("No PDF uploaded yet")
+            with gr.Tabs():
+                with gr.TabItem("PDF"):
+                    pdf_file = gr.File(label="Upload PDF Document", file_types=[".pdf"], type="binary")
+                    pdf_upload_button = gr.Button("Process PDF", variant="primary")
+                
+                with gr.TabItem("Excel"):
+                    excel_file = gr.File(label="Upload Excel File", file_types=[".xlsx", ".xls"], type="binary")
+                    excel_upload_button = gr.Button("Process Excel", variant="primary")
+                
+                with gr.TabItem("Image"):
+                    image_input = gr.File(
+                        label="Upload Image", 
+                        file_types=["image"],
+                        type="filepath"
+                    )
+                    analyze_btn = gr.Button("Analyze Image")
+            
+            file_status = gr.Markdown("No file uploaded yet")
+            
+            # Model selector
             model_dropdown = gr.Dropdown(
                 choices=["llama3-70b-8192", "llama3-8b-8192", "mixtral-8x7b-32768", "gemma-7b-it"],
                 value="llama3-70b-8192",
                 label="Select Groq Model"
             )
-            
-            # Tech Tools Section
-            gr.Markdown("### Developer Tools", elem_classes="tool-title")
-            with gr.Group(elem_classes="tool-container"):
-                with gr.Tabs():
-                    with gr.TabItem("GitHub Search"):
-                        repo_query = gr.Textbox(label="Search Query", placeholder="Enter keywords to search for repositories")
-                        with gr.Row():
-                            language = gr.Dropdown(
-                                choices=["any", "JavaScript", "Python", "Java", "C++", "TypeScript", "Go", "Rust", "PHP", "C#"],
-                                value="any",
-                                label="Language"
-                            )
-                            min_stars = gr.Dropdown(
-                                choices=["0", "10", "50", "100", "1000", "10000"],
-                                value="0",
-                                label="Min Stars"
-                            )
-                        sort_by = gr.Dropdown(
-                            choices=["stars", "forks", "updated"],
-                            value="stars",
-                            label="Sort By"
-                        )
-                        repo_search_btn = gr.Button("Search Repositories")
-                    
-                    with gr.TabItem("Stack Overflow"):
-                        stack_query = gr.Textbox(label="Search Query", placeholder="Enter your technical question")
-                        with gr.Row():
-                            tag = gr.Dropdown(
-                                choices=["any", "python", "javascript", "java", "c++", "react", "node.js", "android", "ios", "sql"],
-                                value="any",
-                                label="Tag"
-                            )
-                            so_sort_by = gr.Dropdown(
-                                choices=["votes", "newest", "activity"],
-                                value="votes",
-                                label="Sort By"
-                            )
-                        so_search_btn = gr.Button("Search Stack Overflow")
-                    
-                    with gr.TabItem("Code Explainer"):
-                        code_input = gr.Textbox(
-                            label="Code to Explain", 
-                            placeholder="Paste your code here...",
-                            lines=10
-                        )
-                        explain_btn = gr.Button("Explain Code")
-                
+        
         with gr.Column(scale=2, min_width=600):
             with gr.Tabs():
                 with gr.TabItem("PDF Viewer"):
                     with gr.Column(elem_classes="pdf-viewer-container"):
                         page_slider = gr.Slider(minimum=1, maximum=1, step=1, label="Page Number", value=1)
                         pdf_image = gr.Image(label="PDF Page", type="pil", elem_classes="pdf-viewer-image")
-                        stats_display = gr.Markdown("No PDF uploaded yet", elem_classes="stats-box")
+                        pdf_stats = gr.Markdown("No PDF uploaded yet", elem_classes="stats-box")
                 
-                with gr.TabItem("GitHub Results"):
-                    repo_results = gr.Markdown("Search for repositories to see results here")
+                with gr.TabItem("Excel Viewer"):
+                    excel_preview = gr.HTML(label="Excel Preview", elem_classes="file-preview")
+                    excel_stats = gr.Markdown("No Excel file uploaded yet", elem_classes="stats-box")
                 
-                with gr.TabItem("Stack Overflow Results"):
-                    stack_results = gr.Markdown("Search for questions to see results here")
-                    
-                with gr.TabItem("Code Explanation"):
-                    code_explanation = gr.Markdown("Paste your code and click 'Explain Code' to see an explanation here")
+                with gr.TabItem("Image Analysis"):
+                    image_preview = gr.Image(label="Image Preview", type="pil")
+                    image_analysis_results = gr.Markdown("Upload an image and click 'Analyze Image' to see analysis results")
+    
+    # Audio visualization elements
+    with gr.Row(elem_classes="container"):
+        with gr.Column():
+            audio_vis = gr.HTML("""
+            <div class="audio-visualization">
+                <div class="audio-bar" style="height: 5px;"></div>
+                <div class="audio-bar" style="height: 12px;"></div>
+                <div class="audio-bar" style="height: 18px;"></div>
+                <div class="audio-bar" style="height: 15px;"></div>
+                <div class="audio-bar" style="height: 10px;"></div>
+                <div class="audio-bar" style="height: 20px;"></div>
+                <div class="audio-bar" style="height: 14px;"></div>
+                <div class="audio-bar" style="height: 8px;"></div>
+            </div>
+            """, visible=False)
+            audio_status_display = gr.Markdown("", elem_classes="audio-status")
     
+    # Chat interface
     with gr.Row(elem_classes="container"):
         with gr.Column(scale=2, min_width=600):
-            chatbot = gr.Chatbot(height=500, bubble_full_width=False, show_copy_button=True, elem_classes="chat-container")
+            chatbot = gr.Chatbot(
+                height=400, 
+                show_copy_button=True, 
+                elem_classes="chat-container",
+                type="messages"  # Use the new messages format
+            )
             with gr.Row():
-                msg = gr.Textbox(show_label=False, placeholder="Ask about your document, type /github to search repos, or /stack to search Stack Overflow...", scale=5)
+                msg = gr.Textbox(
+                    show_label=False, 
+                    placeholder="Ask about your document or click the microphone to speak...", 
+                    scale=5
+                )
+                voice_btn = gr.Button("🎤", elem_classes="voice-btn")
                 send_btn = gr.Button("Send", scale=1)
-            clear_btn = gr.Button("Clear Conversation")
+                
+            with gr.Row(elem_classes="audio-controls"):
+                clear_btn = gr.Button("Clear Conversation")
+                speak_btn = gr.Button("🔊 Speak Response", elem_classes="speak-btn")
+                audio_player = gr.Audio(label="Response Audio", type="filepath", visible=False)
     
-    # Event Handlers
-    upload_button.click(
+    # Event Handlers for PDF processing
+    pdf_upload_button.click(
+        lambda x: ("pdf", x),
+        inputs=[pdf_file],
+        outputs=[file_type, file_status]
+    ).then(
         process_pdf,
         inputs=[pdf_file],
-        outputs=[current_session_id, pdf_status, pdf_state]
+        outputs=[current_session_id, file_status, pdf_state]
     ).then(
         update_pdf_viewer,
         inputs=[pdf_state],
-        outputs=[page_slider, pdf_image, stats_display]
+        outputs=[page_slider, pdf_image, pdf_stats]
+    )
+    
+    # Event Handlers for Excel processing
+    def update_excel_preview(state):
+        if not state:
+            return "", "No Excel file uploaded yet"
+        preview = state.get("data_preview", "")
+        sheets = state.get("total_sheets", 0)
+        rows = state.get("total_rows", 0)
+        stats = f"**Excel Statistics:**\nSheets: {sheets}\nTotal Rows: {rows}"
+        return preview, stats
+    
+    excel_upload_button.click(
+        lambda x: ("excel", x),
+        inputs=[excel_file],
+        outputs=[file_type, file_status]
+    ).then(
+        process_excel,
+        inputs=[excel_file],
+        outputs=[current_session_id, file_status, excel_state]
+    ).then(
+        update_excel_preview,
+        inputs=[excel_state],
+        outputs=[excel_preview, excel_stats]
     )
     
+    # Event Handlers for Image Analysis
+    analyze_btn.click(
+        lambda x: ("image", x),
+        inputs=[image_input],
+        outputs=[file_type, file_status]
+    ).then(
+        analyze_image,
+        inputs=[image_input],
+        outputs=[image_analysis_results]
+    ).then(
+        lambda x: Image.open(x) if x else None,
+        inputs=[image_input],
+        outputs=[image_preview]
+    )
+    
+    # Chat message handling
     msg.submit(
         generate_response,
         inputs=[msg, current_session_id, model_dropdown, chatbot],
@@ -549,44 +842,56 @@ with gr.Blocks(css=custom_css, theme=gr.themes.Soft()) as demo:
         outputs=[chatbot]
     ).then(lambda: "", None, [msg])
     
-    clear_btn.click(
-        lambda: ([], None, "No PDF uploaded yet", {"page_images": [], "total_pages": 0, "total_words": 0}, 0, None, "No PDF uploaded yet"),
-        None,
-        [chatbot, current_session_id, pdf_status, pdf_state, page_slider, pdf_image, stats_display]
+    # Improved speech-to-text with visual feedback
+    voice_btn.click(
+        speech_to_text,
+        inputs=[audio_status],
+        outputs=[audio_status_display, audio_vis, msg]
+    )
+    
+    # Improved text-to-speech with visual feedback
+    speak_btn.click(
+        text_to_speech,
+        inputs=[audio_status, chatbot],
+        outputs=[audio_status_display, audio_vis, audio_player]
+    ).then(
+        lambda x: gr.update(visible=True) if x else gr.update(visible=False),
+        inputs=[audio_player],
+        outputs=[audio_player]
     )
     
+    # Page navigation for PDF
     page_slider.change(
         update_image,
         inputs=[page_slider, pdf_state],
         outputs=[pdf_image]
     )
     
-    # Tech tool handlers
-    repo_search_btn.click(
-        perform_repo_search,
-        inputs=[repo_query, language, sort_by, min_stars],
-        outputs=[repo_results]
-    )
-    
-    so_search_btn.click(
-        perform_stack_search,
-        inputs=[stack_query, tag, so_sort_by],
-        outputs=[stack_results]
-    )
-    
-    explain_btn.click(
-        explain_code,
-        inputs=[code_input],
-        outputs=[code_explanation]
+    # Clear conversation and reset UI
+    clear_btn.click(
+        lambda: (
+            [], None, "No file uploaded yet", 
+            {"page_images": [], "total_pages": 0, "total_words": 0},
+            {"data_preview": "", "total_sheets": 0, "total_rows": 0},
+            "none", 0, None, "No PDF uploaded yet", "", 
+            "No Excel file uploaded yet", None, 
+            "Upload an image and click 'Analyze Image' to see results", None,
+            gr.update(visible=False), "Ready"
+        ),
+        None,
+        [chatbot, current_session_id, file_status, pdf_state, excel_state, 
+         file_type, page_slider, pdf_image, pdf_stats, excel_preview, 
+         excel_stats, image_preview, image_analysis_results, audio_player, 
+         audio_vis, audio_status_display]
     )
 
-# Add footer with attribution
-gr.HTML("""
-<div style="text-align: center; margin-top: 20px; padding: 10px; color: #666; font-size: 0.8rem; border-top: 1px solid #eee;">
-    Created by Calvin Allen Crawford
-</div>
-""")
+    # Add footer with creator attribution
+    gr.HTML("""
+    <div style="text-align: center; margin-top: 20px; padding: 10px; color: #666; font-size: 0.8rem; border-top: 1px solid #eee;">
+        Created by Calvin Allen Crawford
+    </div>
+    """)
 
 # Launch the app
 if __name__ == "__main__":
-    demo.launch()
+    demo.launch()