Update app.py

app.py

@@ -360,16 +360,7 @@ def is_content_unique(new_content, existing_contents, similarity_threshold=0.8):
 
 def assess_relevance_and_summarize(llm_client, query, document, temperature=0.2):
     """
-    Enhanced function to assess document relevance with entity detection and URL analysis.
-    
-    Args:
-        llm_client: The LLM client instance
-        query: User's search query
-        document: Dictionary containing document info (url, content, etc.)
-        temperature: Temperature parameter for LLM
-    
-    Returns:
-        String containing relevance assessment and summary
+    Fixed version of relevance assessment function with more reliable scoring.
     """
     # First, detect entities in the query using LLM
     entity_detection_prompt = """Analyze the following query and identify any specific named entities (companies, people, organizations, products, etc.). Return ONLY the entities, separated by commas. If no entities are found, return 'None'.
@@ -387,53 +378,54 @@ Entities:"""
         entity_response = llm_client.chat_completion(
             messages=entity_messages,
             max_tokens=100,
-            temperature=0.1  # Lower temperature for more consistent entity detection
+            temperature=0.1
         )
         entities = entity_response.choices[0].message.content.strip()
         
-        # Calculate URL relevance score based on entities
+        # Calculate initial URL relevance score
         url_relevance_score = 0
+        url = document['url'].lower()
         if entities.lower() != 'none':
-            url = document['url'].lower()
             for entity in entities.split(','):
                 entity = entity.strip().lower()
                 if entity in url:
                     url_relevance_score += 1
 
-        # Prepare the main assessment prompt with entity and URL information
-        system_prompt = """You are a world class AI assistant specializing in document relevance assessment and summarization. Your task is to:
-1. Consider any detected entities and URL relevance
-2. Assess if the document content is relevant to the user's query
-3. Provide a relevance score and summary if relevant
+        # Prepare the main assessment prompt with explicit scoring rules
+        system_prompt = """You are a world class AI assistant specializing in document relevance assessment. Analyze the document's relevance to the query using this scoring system:
+
+Scoring Rules (Total possible score: 5):
+1. Query Topic Match:
+   - Direct match: 2 points
+   - Partial match: 1 point
+2. Entity Presence:
+   - Contains key entities from query: 1 point
+3. Content Quality:
+   - Recent/timely information: 1 point
+   - Detailed/specific information: 1 point
 
-Use the following scoring criteria:
-- URL contains query entities: +1 point per entity
-- Content directly addresses the query topic: +2 points
-- Content contains relevant but indirect information: +1 point
-- Content is recent and up-to-date (if time-sensitive): +1 point
-- Content provides unique insights: +1 point"""
+You MUST provide a numerical score following these rules."""
 
         user_prompt = f"""
 Query: {query}
 Detected Entities: {entities}
-URL Relevance Score: {url_relevance_score}
+URL Contains Entities Score: {url_relevance_score}
 
 Document Content:
 {document['content']}
 
-Please provide your assessment in the following format:
-Relevant: [Yes/No]
-Relevance Score: [Score out of 5]
-URL Priority: [High if URL contains entities, Low if not]
-Summary: [Your 1-2 sentence summary if relevant, or "Not relevant" if not]
-Entities Mentioned: [List entities from the query that appear in the content]"""
+Provide your assessment in EXACTLY this format:
+Relevant: Yes/No
+Relevance Score: [NUMBER]/5
+URL Priority: {"High" if url_relevance_score > 0 else "Low"}
+Summary: [1-2 sentence summary if relevant, or "Not relevant" if not]
+Entities Mentioned: [List any query entities found in content]"""
 
         messages = [
             {"role": "system", "content": system_prompt},
             {"role": "user", "content": user_prompt}
         ]
 
-        # Get the final assessment
         response = llm_client.chat_completion(
             messages=messages,
             max_tokens=250,
@@ -445,11 +437,74 @@ Entities Mentioned: [List entities from the query that appear in the content]"""
     except Exception as e:
         logger.error(f"Error in enhanced relevance assessment: {e}")
         return f"""Relevant: No
-Relevance Score: 0
+Relevance Score: 0/5
 URL Priority: Low
 Summary: Error during assessment - {str(e)}
 Entities Mentioned: None"""
 
+# Modified processing section for search_and_scrape function
+def process_relevance_assessments(scraped_content, client, rephrased_query, llm_temperature):
+    """
+    Separate function for processing relevance assessments with fixed scoring handling.
+    """
+    relevant_documents = []
+    unique_summaries = []
+    
+    for doc in scraped_content:
+        assessment = assess_relevance_and_summarize(client, rephrased_query, doc, temperature=llm_temperature)
+        
+        # Parse the structured assessment response
+        assessment_parts = {}
+        for line in assessment.split('\n'):
+            if ':' in line:
+                key, value = line.split(':', 1)
+                assessment_parts[key.strip()] = value.strip()
+        
+        # Extract and properly parse the relevance score
+        try:
+            relevance_score_str = assessment_parts.get('Relevance Score', '0/5')
+            relevance_score = float(relevance_score_str.split('/')[0])
+        except (ValueError, IndexError):
+            relevance_score = 0
+            logger.warning(f"Failed to parse relevance score: {relevance_score_str}")
+        
+        is_relevant = assessment_parts.get('Relevant', '').lower() == 'yes'
+        url_priority = assessment_parts.get('URL Priority', 'Low')
+        summary_text = assessment_parts.get('Summary', '').strip()
+        entities_mentioned = assessment_parts.get('Entities Mentioned', 'None')
+        
+        # Lower the threshold to catch more potentially relevant documents
+        RELEVANCE_THRESHOLD = 1.0  # Lowered from 2.5
+        
+        if is_relevant or relevance_score >= RELEVANCE_THRESHOLD:
+            # Check for content uniqueness
+            if is_content_unique(summary_text, unique_summaries):
+                doc_record = {
+                    "title": doc['title'],
+                    "url": doc['url'],
+                    "summary": summary_text,
+                    "scraper": doc['scraper'],
+                    "relevance_score": relevance_score,
+                    "url_priority": url_priority,
+                    "entities_mentioned": entities_mentioned
+                }
+                
+                relevant_documents.append(doc_record)
+                unique_summaries.append(summary_text)
+                logger.info(f"Added relevant document: {doc['title']} (Score: {relevance_score}, Priority: {url_priority})")
+            else:
+                logger.info(f"Skipping similar content: {doc['title']}")
+        else:
+            logger.info(f"Skipping document: {doc['title']} (Score: {relevance_score})")
+    
+    # Sort by both URL priority and relevance score
+    relevant_documents.sort(key=lambda x: (
+        x['url_priority'] == 'High',
+        x['relevance_score']
+    ), reverse=True)
+    
+    return relevant_documents
+
 def scrape_full_content(url, scraper="bs4", max_chars=3000, timeout=5):
     try:
         logger.info(f"Scraping full content from: {url}")
@@ -653,65 +708,13 @@ def search_and_scrape(query, chat_history, num_results=5, scraper="bs4", max_cha
         logger.info(f"Successfully scraped {len(scraped_content)} documents.")
 
         # Step 3: Assess relevance, summarize, and check for uniqueness
-        relevant_documents = []
-        unique_summaries = []
-        
-        # Sort scraped_content based on initial URL analysis (if entities are in URL)
-        for doc in scraped_content:
-            assessment = assess_relevance_and_summarize(client, rephrased_query, doc, temperature=llm_temperature)
-            
-            # Parse the structured assessment response
-            assessment_parts = {}
-            for line in assessment.split('\n'):
-                if ':' in line:
-                    key, value = line.split(':', 1)
-                    assessment_parts[key.strip()] = value.strip()
-            
-            # Extract relevant information
-            is_relevant = assessment_parts.get('Relevant', 'No').lower() == 'yes'
-            relevance_score = float(assessment_parts.get('Relevance Score', '0').split('/')[0])
-            url_priority = assessment_parts.get('URL Priority', 'Low')
-            summary_text = assessment_parts.get('Summary', 'Not relevant')
-            entities_mentioned = assessment_parts.get('Entities Mentioned', 'None')
-            
-            # Define relevance threshold
-            RELEVANCE_THRESHOLD = 2.5  # Documents must score above 2.5 out of 5 to be considered
-            
-            if is_relevant and relevance_score >= RELEVANCE_THRESHOLD:
-                # Check for content uniqueness
-                if is_content_unique(summary_text, unique_summaries):
-                    # Create enhanced document record
-                    doc_record = {
-                        "title": doc['title'],
-                        "url": doc['url'],
-                        "summary": summary_text,
-                        "scraper": doc['scraper'],
-                        "relevance_score": relevance_score,
-                        "url_priority": url_priority,
-                        "entities_mentioned": entities_mentioned,
-                        "original_content": doc.get('content', '')  # Keep original content if needed
-                    }
-                    
-                    relevant_documents.append(doc_record)
-                    unique_summaries.append(summary_text)
-                    logger.info(f"Added relevant document: {doc['title']} (Score: {relevance_score}, Priority: {url_priority})")
-                else:
-                    logger.info(f"Skipping similar content: {doc['title']}")
-            else:
-                logger.info(f"Skipping irrelevant or low-scoring document: {doc['title']} (Score: {relevance_score})")
-        
-        # Sort relevant documents by relevance score and URL priority
-        relevant_documents.sort(key=lambda x: (
-            x['url_priority'] == 'High',  # True sorts before False
-            x['relevance_score']
-        ), reverse=True)
+        relevant_documents = process_relevance_assessments(scraped_content, client, rephrased_query, llm_temperature)
         
         if not relevant_documents:
             logger.warning("No relevant and unique documents found.")
             return "No relevant and unique documents found for the given query."
         
         logger.info(f"Found {len(relevant_documents)} relevant and unique documents")
-        logger.debug(f"Top document scores: {[(doc['title'], doc['relevance_score']) for doc in relevant_documents[:3]]}")
 
         # Step 4: Rerank documents based on similarity to query
         reranked_docs = rerank_documents(rephrased_query, relevant_documents)