Spaces:
Build error
Build error
| import streamlit as st | |
| from typing import List, Dict | |
| import httpx | |
| from pathlib import Path | |
| import os | |
| from dotenv import load_dotenv | |
| import json | |
| import numpy as np | |
| from pymongo import MongoClient | |
| from openai import OpenAI | |
| from datetime import datetime | |
| import asyncio | |
| import pandas as pd | |
| # Load environment variables | |
| load_dotenv() | |
| PERPLEXITY_API_KEY = os.getenv("PERPLEXITY_KEY") | |
| MONGODB_URI = os.getenv("MONGO_URI") | |
| OPENAI_API_KEY = os.getenv("OPENAI_KEY") | |
| # Initialize MongoDB client | |
| client = MongoClient(MONGODB_URI) | |
| db = client["document_analysis"] | |
| vectors_collection = db["document_vectors"] | |
| # Initialize OpenAI client | |
| openai_client = OpenAI(api_key=OPENAI_API_KEY) | |
| class GoalAnalyzer: | |
| def __init__(self): | |
| self.api_key = PERPLEXITY_API_KEY | |
| self.base_url = "https://api.perplexity.ai/chat/completions" | |
| def clean_json_string(self, content: str) -> str: | |
| """Clean and extract valid JSON from string""" | |
| # Remove markdown formatting | |
| if "```json" in content: | |
| content = content.split("```json")[1].split("```")[0] | |
| elif "```" in content: | |
| content = content.split("```")[1] | |
| # Find the JSON object boundaries | |
| start_idx = content.find("{") | |
| end_idx = content.rfind("}") + 1 | |
| if start_idx != -1 and end_idx > 0: | |
| content = content[start_idx:end_idx] | |
| # Clean up common issues | |
| content = content.strip() | |
| content = content.replace("\n", "") | |
| content = content.replace("'", '"') | |
| return content | |
| async def get_perplexity_analysis(self, text: str, goal: str) -> Dict: | |
| """Get analysis from Perplexity API""" | |
| headers = { | |
| "Authorization": f"Bearer {self.api_key}", | |
| "Content-Type": "application/json", | |
| } | |
| prompt = f""" | |
| Analyze the following text in context of the goal: {goal} | |
| Text: {text} | |
| Provide analysis in the following JSON format: | |
| {{ | |
| "themes": ["theme1", "theme2"], | |
| "subthemes": {{"theme1": ["subtheme1", "subtheme2"], "theme2": ["subtheme3"]}}, | |
| "keywords": ["keyword1", "keyword2"], | |
| "relevance_score": 0-100 | |
| }} | |
| """ | |
| try: | |
| async with httpx.AsyncClient() as client: | |
| payload = { | |
| "model": "llama-3.1-sonar-small-128k-chat", # Updated to supported model | |
| "messages": [ | |
| { | |
| "role": "system", | |
| "content": "You are an AI assistant that analyzes documents and provides structured analysis.", | |
| }, | |
| {"role": "user", "content": prompt}, | |
| ], | |
| "max_tokens": 1024, | |
| } | |
| # Debug info using expander | |
| with st.expander("Debug Info", expanded=False): | |
| st.write("Request payload:", payload) | |
| response = await client.post( | |
| self.base_url, headers=headers, json=payload, timeout=30.0 | |
| ) | |
| # Debug response info | |
| with st.expander("Response Info", expanded=False): | |
| st.write("Response status:", response.status_code) | |
| st.write("Response headers:", dict(response.headers)) | |
| st.write("Response content:", response.text) | |
| if response.status_code != 200: | |
| error_detail = ( | |
| response.json() if response.content else "No error details" | |
| ) | |
| raise Exception( | |
| f"API returned status code {response.status_code}. Details: {error_detail}" | |
| ) | |
| result = response.json() | |
| content = ( | |
| result.get("choices", [{}])[0].get("message", {}).get("content", "") | |
| ) | |
| # Clean and parse JSON | |
| cleaned_content = self.clean_json_string(content) | |
| try: | |
| analysis = json.loads(cleaned_content) | |
| # Validate required fields | |
| required_fields = [ | |
| "themes", | |
| "subthemes", | |
| "keywords", | |
| "relevance_score", | |
| ] | |
| for field in required_fields: | |
| if field not in analysis: | |
| analysis[field] = [] if field != "relevance_score" else 0 | |
| return analysis | |
| except json.JSONDecodeError as e: | |
| st.error(f"JSON parsing error: {str(e)}") | |
| st.error(f"Failed content: {cleaned_content}") | |
| return { | |
| "themes": ["Error parsing themes"], | |
| "subthemes": {"Error": ["Failed to parse subthemes"]}, | |
| "keywords": ["parsing-error"], | |
| "relevance_score": 0, | |
| } | |
| except Exception as e: | |
| st.error(f"API Error: {str(e)}") | |
| return None | |
| def extract_text_from_file(self, file) -> str: | |
| """Extract text content from uploaded file""" | |
| try: | |
| text = "" | |
| file_type = file.type | |
| if file_type == "text/plain": | |
| text = file.getvalue().decode("utf-8") | |
| elif file_type == "application/pdf": | |
| import PyPDF2 | |
| pdf_reader = PyPDF2.PdfReader(file) | |
| for page in pdf_reader.pages: | |
| text += page.extract_text() | |
| elif ( | |
| file_type | |
| == "application/vnd.openxmlformats-officedocument.wordprocessingml.document" | |
| ): | |
| import docx | |
| doc = docx.Document(file) | |
| text = " ".join([paragraph.text for paragraph in doc.paragraphs]) | |
| return text | |
| except Exception as e: | |
| st.error(f"Error extracting text: {str(e)}") | |
| return "" | |
| class DocumentVectorizer: | |
| def __init__(self): | |
| self.model = "text-embedding-ada-002" | |
| self.client = MongoClient(MONGODB_URI) | |
| self.db = self.client["document_analysis"] | |
| self.vectors_collection = self.db["document_vectors"] | |
| # Create vector search index if it doesn't exist | |
| try: | |
| self.vectors_collection.create_index( | |
| [("vector", "2dsphere")], # Changed to 2dsphere for vector indexing | |
| { | |
| "vectorSearchConfig": { | |
| "dimensions": 1536, # OpenAI embedding dimensions | |
| "similarity": "cosine", | |
| } | |
| }, | |
| ) | |
| except Exception as e: | |
| st.warning(f"Vector index may already exist") | |
| def get_embedding(self, text: str) -> list: | |
| """Get embedding vector for text using OpenAI""" | |
| try: | |
| response = openai_client.embeddings.create(model=self.model, input=text) | |
| return response.data[0].embedding | |
| except Exception as e: | |
| st.error(f"Error getting embedding: {str(e)}") | |
| return None | |
| # Add this method to DocumentVectorizer class | |
| def vector_exists(self, doc_name: str) -> bool: | |
| """Check if vector exists for document""" | |
| return self.vectors_collection.count_documents({"name": doc_name}) > 0 | |
| # Update store_vector method in DocumentVectorizer class | |
| def store_vector(self, doc_name: str, vector: list, text: str, goal: str = None): | |
| """Store document/goal vector in MongoDB using upsert""" | |
| try: | |
| vector_doc = { | |
| "name": doc_name, | |
| "vector": vector, | |
| "text": text, | |
| "type": "document" if goal is None else "goal", | |
| "goal": goal, | |
| "updated_at": datetime.utcnow(), | |
| } | |
| # Use update_one with upsert | |
| self.vectors_collection.update_one( | |
| {"name": doc_name}, | |
| {"$set": vector_doc, "$setOnInsert": {"created_at": datetime.utcnow()}}, | |
| upsert=True, | |
| ) | |
| except Exception as e: | |
| st.error(f"Error storing vector: {str(e)}") | |
| # Update vector_search method in DocumentVectorizer class | |
| def vector_search(self, query_vector: List[float], limit: int = 5) -> List[Dict]: | |
| """Search for similar documents using vector similarity""" | |
| try: | |
| # Get all documents | |
| documents = list(self.vectors_collection.find({"type": "document"})) | |
| # Calculate similarities | |
| similarities = [] | |
| for doc in documents: | |
| similarity = self.calculate_similarity(query_vector, doc["vector"]) | |
| similarities.append( | |
| { | |
| "name": doc["name"], | |
| "text": doc["text"], | |
| "similarity": similarity, # Keep as float | |
| "similarity_display": f"{similarity*100:.1f}%", # Add display version | |
| } | |
| ) | |
| # Sort by similarity and get top k | |
| sorted_docs = sorted( | |
| similarities, | |
| key=lambda x: x["similarity"], # Sort by float value | |
| reverse=True, | |
| )[:limit] | |
| return sorted_docs | |
| except Exception as e: | |
| st.error(f"Vector search error: {str(e)}") | |
| return [] | |
| def find_similar_documents(self, text: str, limit: int = 5) -> List[Dict]: | |
| """Find similar documents for given text""" | |
| vector = self.get_embedding(text) | |
| if vector: | |
| return self.vector_search(vector, limit) | |
| return [] | |
| def calculate_similarity(self, vector1: list, vector2: list) -> float: | |
| """Calculate cosine similarity between two vectors""" | |
| return np.dot(vector1, vector2) / ( | |
| np.linalg.norm(vector1) * np.linalg.norm(vector2) | |
| ) | |
| def display_analysis_results(analysis: Dict): | |
| """Display analysis results in Streamlit UI""" | |
| if not analysis: | |
| return | |
| # Display Themes | |
| st.subheader("Themes") | |
| for theme in analysis.get("themes", []): | |
| with st.expander(f"🎯 {theme}"): | |
| # Display subthemes for this theme | |
| subthemes = analysis.get("subthemes", {}).get(theme, []) | |
| if subthemes: | |
| st.write("**Subthemes:**") | |
| for subtheme in subthemes: | |
| st.write(f"- {subtheme}") | |
| # Display Keywords | |
| st.subheader("Keywords") | |
| keywords = analysis.get("keywords", []) | |
| st.write(" | ".join([f"🔑 {keyword}" for keyword in keywords])) | |
| # Display Relevance Score | |
| score = analysis.get("relevance_score", 0) | |
| st.metric("Relevance Score", f"{score}%") | |
| def display_analyst_dashboard(): | |
| st.title("Multi-Goal Document Analysis") | |
| with st.sidebar: | |
| st.markdown("### Input Section") | |
| tab1, tab2 = st.tabs(["Document Analysis", "Similarity Search"]) | |
| # tab1, tab2 = st.tabs(["Document Analysis", "Similarity Search"]) | |
| with tab1: | |
| # Multiple goals input | |
| num_goals = st.number_input("Number of goals:", min_value=1, value=1) | |
| goals = [] | |
| for i in range(num_goals): | |
| goal = st.text_area(f"Goal {i+1}:", key=f"goal_{i}", height=100) | |
| if goal: | |
| goals.append(goal) | |
| uploaded_files = st.file_uploader( | |
| "Upload documents", | |
| accept_multiple_files=True, | |
| type=["txt", "pdf", "docx"], | |
| ) | |
| analyze_button = ( | |
| st.button("Analyze Documents") if goals and uploaded_files else None | |
| ) | |
| with tab2: | |
| # Keep existing similarity search tab | |
| search_text = st.text_area("Enter text to find similar documents:") | |
| search_limit = st.slider("Number of results", 1, 10, 5) | |
| search_button = st.button("Search Similar") if search_text else None | |
| if st.button("Logout", use_container_width=True): | |
| for key in st.session_state.keys(): | |
| del st.session_state[key] | |
| st.rerun() | |
| if analyze_button: | |
| analyzer = GoalAnalyzer() | |
| vectorizer = DocumentVectorizer() | |
| # Store vectors | |
| doc_vectors = {} | |
| goal_vectors = {} | |
| # Process goals first | |
| with st.spinner("Processing goals..."): | |
| for i, goal in enumerate(goals): | |
| vector = vectorizer.get_embedding(goal) | |
| if vector: | |
| goal_vectors[f"Goal {i+1}"] = vector | |
| vectorizer.store_vector(f"Goal {i+1}", vector, goal, goal) | |
| # Process documents | |
| with st.spinner("Processing documents..."): | |
| for file in uploaded_files: | |
| st.markdown(f"### Analysis for {file.name}") | |
| if vectorizer.vector_exists(file.name): | |
| st.info(f"Vector already exists for {file.name}") | |
| existing_doc = vectorizer.vectors_collection.find_one( | |
| {"name": file.name} | |
| ) | |
| doc_vectors[file.name] = existing_doc["vector"] | |
| else: | |
| text = analyzer.extract_text_from_file(file) | |
| if not text: | |
| st.warning(f"Could not extract text from {file.name}") | |
| continue | |
| vector = vectorizer.get_embedding(text) | |
| if vector: | |
| doc_vectors[file.name] = vector | |
| vectorizer.store_vector(file.name, vector, text) | |
| # Display goal similarities | |
| st.subheader("Goal Relevance Scores") | |
| col1, col2 = st.columns([1, 2]) | |
| with col1: | |
| for goal_name, goal_vector in goal_vectors.items(): | |
| similarity = ( | |
| vectorizer.calculate_similarity( | |
| doc_vectors[file.name], goal_vector | |
| ) | |
| * 100 | |
| ) | |
| st.metric(f"{goal_name}", f"{similarity:.1f}%") | |
| with col2: | |
| # Get analysis for all goals combined | |
| analysis = asyncio.run( | |
| analyzer.get_perplexity_analysis(text, " | ".join(goals)) | |
| ) | |
| display_analysis_results(analysis) | |
| st.divider() | |
| # Document similarity matrix | |
| if len(doc_vectors) > 1: | |
| st.markdown("### Document Similarity Matrix") | |
| files = list(doc_vectors.keys()) | |
| similarity_matrix = [] | |
| for file1 in files: | |
| row = [] | |
| for file2 in files: | |
| similarity = vectorizer.calculate_similarity( | |
| doc_vectors[file1], doc_vectors[file2] | |
| ) | |
| row.append(similarity) | |
| similarity_matrix.append(row) | |
| df = pd.DataFrame(similarity_matrix, columns=files, index=files) | |
| st.dataframe(df.style.background_gradient(cmap="RdYlGn")) | |
| # Add goal-document similarity matrix | |
| st.markdown("### Goal-Document Similarity Matrix") | |
| goal_doc_matrix = [] | |
| goal_names = list(goal_vectors.keys()) | |
| for file in files: | |
| row = [] | |
| for goal in goal_names: | |
| similarity = vectorizer.calculate_similarity( | |
| doc_vectors[file], goal_vectors[goal] | |
| ) | |
| row.append(similarity) | |
| goal_doc_matrix.append(row) | |
| df_goals = pd.DataFrame( | |
| goal_doc_matrix, columns=goal_names, index=files | |
| ) | |
| st.dataframe(df_goals.style.background_gradient(cmap="RdYlGn")) | |
| # Keep existing similarity search functionality | |
| elif search_button: | |
| vectorizer = DocumentVectorizer() | |
| with st.spinner("Searching similar documents..."): | |
| query_vector = vectorizer.get_embedding(search_text) | |
| if query_vector: | |
| similar_docs = vectorizer.vector_search(query_vector, search_limit) | |
| if similar_docs: | |
| st.markdown("### Similar Documents Found") | |
| # Create DataFrame with numeric similarities | |
| df = pd.DataFrame(similar_docs) | |
| # Apply gradient to numeric column | |
| styled_df = df[["name", "similarity"]].style.background_gradient( | |
| cmap="RdYlGn", subset=["similarity"] | |
| ) | |
| # Format display after styling | |
| styled_df = styled_df.format({"similarity": "{:.1%}"}) | |
| st.dataframe(styled_df) | |
| # Show document contents | |
| for doc in similar_docs: | |
| with st.expander( | |
| f"📄 {doc['name']} (Similarity: {doc['similarity_display']})" | |
| ): | |
| st.text( | |
| doc["text"][:20] + "..." | |
| if len(doc["text"]) > 20 | |
| else doc["text"] | |
| ) | |
| else: | |
| st.info("No similar documents found") | |
| else: | |
| st.error("Could not process search query") | |
| def main(): | |
| st.title("Multi-Goal Document Analysis") | |
| with st.sidebar: | |
| st.markdown("### Input Section") | |
| tab1, tab2 = st.tabs(["Document Analysis", "Similarity Search"]) | |
| # tab1, tab2 = st.tabs(["Document Analysis", "Similarity Search"]) | |
| with tab1: | |
| # Multiple goals input | |
| num_goals = st.number_input("Number of goals:", min_value=1, value=1) | |
| goals = [] | |
| for i in range(num_goals): | |
| goal = st.text_area(f"Goal {i+1}:", key=f"goal_{i}", height=100) | |
| if goal: | |
| goals.append(goal) | |
| uploaded_files = st.file_uploader( | |
| "Upload documents", | |
| accept_multiple_files=True, | |
| type=["txt", "pdf", "docx"], | |
| ) | |
| analyze_button = ( | |
| st.button("Analyze Documents") if goals and uploaded_files else None | |
| ) | |
| with tab2: | |
| # Keep existing similarity search tab | |
| search_text = st.text_area("Enter text to find similar documents:") | |
| search_limit = st.slider("Number of results", 1, 10, 5) | |
| search_button = st.button("Search Similar") if search_text else None | |
| if analyze_button: | |
| analyzer = GoalAnalyzer() | |
| vectorizer = DocumentVectorizer() | |
| # Store vectors | |
| doc_vectors = {} | |
| goal_vectors = {} | |
| # Process goals first | |
| with st.spinner("Processing goals..."): | |
| for i, goal in enumerate(goals): | |
| vector = vectorizer.get_embedding(goal) | |
| if vector: | |
| goal_vectors[f"Goal {i+1}"] = vector | |
| vectorizer.store_vector(f"Goal {i+1}", vector, goal, goal) | |
| # Process documents | |
| with st.spinner("Processing documents..."): | |
| for file in uploaded_files: | |
| st.markdown(f"### Analysis for {file.name}") | |
| if vectorizer.vector_exists(file.name): | |
| st.info(f"Vector already exists for {file.name}") | |
| existing_doc = vectorizer.vectors_collection.find_one( | |
| {"name": file.name} | |
| ) | |
| doc_vectors[file.name] = existing_doc["vector"] | |
| else: | |
| text = analyzer.extract_text_from_file(file) | |
| if not text: | |
| st.warning(f"Could not extract text from {file.name}") | |
| continue | |
| vector = vectorizer.get_embedding(text) | |
| if vector: | |
| doc_vectors[file.name] = vector | |
| vectorizer.store_vector(file.name, vector, text) | |
| # Display goal similarities | |
| st.subheader("Goal Relevance Scores") | |
| col1, col2 = st.columns([1, 2]) | |
| with col1: | |
| for goal_name, goal_vector in goal_vectors.items(): | |
| similarity = ( | |
| vectorizer.calculate_similarity( | |
| doc_vectors[file.name], goal_vector | |
| ) | |
| * 100 | |
| ) | |
| st.metric(f"{goal_name}", f"{similarity:.1f}%") | |
| with col2: | |
| # Get analysis for all goals combined | |
| analysis = asyncio.run( | |
| analyzer.get_perplexity_analysis(text, " | ".join(goals)) | |
| ) | |
| display_analysis_results(analysis) | |
| st.divider() | |
| # Document similarity matrix | |
| if len(doc_vectors) > 1: | |
| st.markdown("### Document Similarity Matrix") | |
| files = list(doc_vectors.keys()) | |
| similarity_matrix = [] | |
| for file1 in files: | |
| row = [] | |
| for file2 in files: | |
| similarity = vectorizer.calculate_similarity( | |
| doc_vectors[file1], doc_vectors[file2] | |
| ) | |
| row.append(similarity) | |
| similarity_matrix.append(row) | |
| df = pd.DataFrame(similarity_matrix, columns=files, index=files) | |
| st.dataframe(df.style.background_gradient(cmap="RdYlGn")) | |
| # Add goal-document similarity matrix | |
| st.markdown("### Goal-Document Similarity Matrix") | |
| goal_doc_matrix = [] | |
| goal_names = list(goal_vectors.keys()) | |
| for file in files: | |
| row = [] | |
| for goal in goal_names: | |
| similarity = vectorizer.calculate_similarity( | |
| doc_vectors[file], goal_vectors[goal] | |
| ) | |
| row.append(similarity) | |
| goal_doc_matrix.append(row) | |
| df_goals = pd.DataFrame( | |
| goal_doc_matrix, columns=goal_names, index=files | |
| ) | |
| st.dataframe(df_goals.style.background_gradient(cmap="RdYlGn")) | |
| # Keep existing similarity search functionality | |
| elif search_button: | |
| vectorizer = DocumentVectorizer() | |
| with st.spinner("Searching similar documents..."): | |
| query_vector = vectorizer.get_embedding(search_text) | |
| if query_vector: | |
| similar_docs = vectorizer.vector_search(query_vector, search_limit) | |
| if similar_docs: | |
| st.markdown("### Similar Documents Found") | |
| # Create DataFrame with numeric similarities | |
| df = pd.DataFrame(similar_docs) | |
| # Apply gradient to numeric column | |
| styled_df = df[["name", "similarity"]].style.background_gradient( | |
| cmap="RdYlGn", subset=["similarity"] | |
| ) | |
| # Format display after styling | |
| styled_df = styled_df.format({"similarity": "{:.1%}"}) | |
| st.dataframe(styled_df) | |
| # Show document contents | |
| for doc in similar_docs: | |
| with st.expander( | |
| f"📄 {doc['name']} (Similarity: {doc['similarity_display']})" | |
| ): | |
| st.text( | |
| doc["text"][:20] + "..." | |
| if len(doc["text"]) > 20 | |
| else doc["text"] | |
| ) | |
| else: | |
| st.info("No similar documents found") | |
| else: | |
| st.error("Could not process search query") | |
| if __name__ == "__main__": | |
| main() | |