Update app.py

app.py

@@ -359,151 +359,42 @@ def is_content_unique(new_content, existing_contents, similarity_threshold=0.8):
     return True
 
 def assess_relevance_and_summarize(llm_client, query, document, temperature=0.2):
-    """
-    Fixed version of relevance assessment function with more reliable scoring.
-    """
-    # First, detect entities in the query using LLM
-    entity_detection_prompt = """Analyze the following query and identify any specific named entities (companies, people, organizations, products, etc.). Return ONLY the entities, separated by commas. If no entities are found, return 'None'.
+    system_prompt = """You are a world class AI assistant. Your task is to assess whether the given text is relevant to the user's query and provide a brief summary if it is relevant."""
 
+    user_prompt = f"""
 Query: {query}
 
-Entities:"""
-
-    entity_messages = [
-        {"role": "system", "content": "You are an expert at identifying named entities in text."},
-        {"role": "user", "content": entity_detection_prompt.format(query=query)}
-    ]
-
-    try:
-        entity_response = llm_client.chat_completion(
-            messages=entity_messages,
-            max_tokens=100,
-            temperature=0.1
-        )
-        entities = entity_response.choices[0].message.content.strip()
-        
-        # Calculate initial URL relevance score
-        url_relevance_score = 0
-        url = document['url'].lower()
-        if entities.lower() != 'none':
-            for entity in entities.split(','):
-                entity = entity.strip().lower()
-                if entity in url:
-                    url_relevance_score += 1
-
-        # Prepare the main assessment prompt with explicit scoring rules
-        system_prompt = """You are a world class AI assistant specializing in document relevance assessment. Analyze the document's relevance to the query using this scoring system:
-
-Scoring Rules (Total possible score: 5):
-1. Query Topic Match:
-   - Direct match: 2 points
-   - Partial match: 1 point
-2. Entity Presence:
-   - Contains key entities from query: 1 point
-3. Content Quality:
-   - Recent/timely information: 1 point
-   - Detailed/specific information: 1 point
-
-You MUST provide a numerical score following these rules."""
-
-        user_prompt = f"""
-Query: {query}
-Detected Entities: {entities}
-URL Contains Entities Score: {url_relevance_score}
-
 Document Content:
 {document['content']}
 
-Provide your assessment in EXACTLY this format:
-Relevant: Yes/No
-Relevance Score: [NUMBER]/5
-URL Priority: {"High" if url_relevance_score > 0 else "Low"}
-Summary: [1-2 sentence summary if relevant, or "Not relevant" if not]
-Entities Mentioned: [List any query entities found in content]"""
+Instructions:
+1. Assess if the document is relevant to the QUERY  made by the user.
+2. If relevant, summarize the main points in 1-2 sentences.
+3. If not relevant, simply state "Not relevant".
 
-        messages = [
-            {"role": "system", "content": system_prompt},
-            {"role": "user", "content": user_prompt}
-        ]
+Your response should be in the following format:
+Relevant: [Yes/No]
+Summary: [Your 1-2 sentence summary if relevant, or "Not relevant" if not]
 
+Remember to focus on financial aspects and implications in your assessment and summary.
+"""
+
+    messages = [
+        {"role": "system", "content": system_prompt},
+        {"role": "user", "content": user_prompt}
+    ]
+
+    try:
         response = llm_client.chat_completion(
             messages=messages,
-            max_tokens=250,
-            temperature=temperature
+            max_tokens=150,
+            temperature=temperature,
+            top_p=0.9
         )
-        
         return response.choices[0].message.content.strip()
-
     except Exception as e:
-        logger.error(f"Error in enhanced relevance assessment: {e}")
-        return f"""Relevant: No
-Relevance Score: 0/5
-URL Priority: Low
-Summary: Error during assessment - {str(e)}
-Entities Mentioned: None"""
-
-# Modified processing section for search_and_scrape function
-def process_relevance_assessments(scraped_content, client, rephrased_query, llm_temperature):
-    """
-    Separate function for processing relevance assessments with fixed scoring handling.
-    """
-    relevant_documents = []
-    unique_summaries = []
-    
-    for doc in scraped_content:
-        assessment = assess_relevance_and_summarize(client, rephrased_query, doc, temperature=llm_temperature)
-        
-        # Parse the structured assessment response
-        assessment_parts = {}
-        for line in assessment.split('\n'):
-            if ':' in line:
-                key, value = line.split(':', 1)
-                assessment_parts[key.strip()] = value.strip()
-        
-        # Extract and properly parse the relevance score
-        try:
-            relevance_score_str = assessment_parts.get('Relevance Score', '0/5')
-            relevance_score = float(relevance_score_str.split('/')[0])
-        except (ValueError, IndexError):
-            relevance_score = 0
-            logger.warning(f"Failed to parse relevance score: {relevance_score_str}")
-        
-        is_relevant = assessment_parts.get('Relevant', '').lower() == 'yes'
-        url_priority = assessment_parts.get('URL Priority', 'Low')
-        summary_text = assessment_parts.get('Summary', '').strip()
-        entities_mentioned = assessment_parts.get('Entities Mentioned', 'None')
-        
-        # Lower the threshold to catch more potentially relevant documents
-        RELEVANCE_THRESHOLD = 1.0  # Lowered from 2.5
-        
-        if is_relevant or relevance_score >= RELEVANCE_THRESHOLD:
-            # Check for content uniqueness
-            if is_content_unique(summary_text, unique_summaries):
-                doc_record = {
-                    "title": doc['title'],
-                    "url": doc['url'],
-                    "summary": summary_text,
-                    "scraper": doc['scraper'],
-                    "relevance_score": relevance_score,
-                    "url_priority": url_priority,
-                    "entities_mentioned": entities_mentioned
-                }
-                
-                relevant_documents.append(doc_record)
-                unique_summaries.append(summary_text)
-                logger.info(f"Added relevant document: {doc['title']} (Score: {relevance_score}, Priority: {url_priority})")
-            else:
-                logger.info(f"Skipping similar content: {doc['title']}")
-        else:
-            logger.info(f"Skipping document: {doc['title']} (Score: {relevance_score})")
-    
-    # Sort by both URL priority and relevance score
-    relevant_documents.sort(key=lambda x: (
-        x['url_priority'] == 'High',
-        x['relevance_score']
-    ), reverse=True)
-    
-    return relevant_documents
+        logger.error(f"Error assessing relevance and summarizing with LLM: {e}")
+        return "Error: Unable to assess relevance and summarize"
 
 def scrape_full_content(url, scraper="bs4", max_chars=3000, timeout=5):
     try:
@@ -708,13 +599,30 @@ def search_and_scrape(query, chat_history, num_results=5, scraper="bs4", max_cha
         logger.info(f"Successfully scraped {len(scraped_content)} documents.")
 
         # Step 3: Assess relevance, summarize, and check for uniqueness
-        relevant_documents = process_relevance_assessments(scraped_content, client, rephrased_query, llm_temperature)
-        
+        relevant_documents = []
+        unique_summaries = []
+        for doc in scraped_content:
+            assessment = assess_relevance_and_summarize(client, rephrased_query, doc, temperature=llm_temperature)
+            relevance, summary = assessment.split('\n', 1)
+            
+            if relevance.strip().lower() == "relevant: yes":
+                summary_text = summary.replace("Summary: ", "").strip()
+                
+                if is_content_unique(summary_text, unique_summaries):
+                    relevant_documents.append({
+                        "title": doc['title'],
+                        "url": doc['url'],
+                        "summary": summary_text,
+                        "scraper": doc['scraper']
+                    })
+                    unique_summaries.append(summary_text)
+                else:
+                    logger.info(f"Skipping similar content: {doc['title']}")
+
         if not relevant_documents:
             logger.warning("No relevant and unique documents found.")
-            return "No relevant and unique documents found for the given query."
-        
-        logger.info(f"Found {len(relevant_documents)} relevant and unique documents")
+            return "No relevant and unique financial news found for the given query."
+            logger.debug(f"Assessment result: {assessment}")
 
         # Step 4: Rerank documents based on similarity to query
         reranked_docs = rerank_documents(rephrased_query, relevant_documents)