Update app.py

app.py

@@ -1,5 +1,6 @@
 # ------------------------------
-# Enhanced NeuroResearch AI System with Refinement Counter and Increased Recursion Limit
+# Enhanced NeuroResearch AI System with Refinement Counter,
+# Dynamic Difficulty Gradient, and Meta-Refinement Inspired by LADDER
 # ------------------------------
 import logging
 import os
@@ -96,7 +97,7 @@ class QuantumDocumentManager:
             self.client = chromadb.PersistentClient(path=ResearchConfig.CHROMA_PATH)
             logger.info("Initialized PersistentClient for Chroma.")
         except Exception as e:
-            logger.error(f"Error initializing PersistentClient: {e}")
+            logger.exception("Error initializing PersistentClient; falling back to in-memory client.")
             self.client = chromadb.Client()  # Fallback to in-memory client
         self.embeddings = OpenAIEmbeddings(
             model="text-embedding-3-large",
@@ -116,7 +117,7 @@ class QuantumDocumentManager:
             docs = splitter.create_documents(documents)
             logger.info(f"Created {len(docs)} document chunks for collection '{collection_name}'.")
         except Exception as e:
-            logger.error(f"Error splitting documents: {e}")
+            logger.exception("Error during document splitting.")
             raise e
 
         return Chroma.from_documents(
@@ -166,7 +167,7 @@ class ResearchRetriever:
             )
             logger.info("Initialized retrievers for research and development domains.")
         except Exception as e:
-            logger.error(f"Error initializing retrievers: {e}")
+            logger.exception("Error initializing retrievers.")
             raise e
 
     def retrieve(self, query: str, domain: str) -> List[Any]:
@@ -182,7 +183,7 @@ class ResearchRetriever:
                 logger.warning(f"Domain '{domain}' not recognized.")
                 return []
         except Exception as e:
-            logger.error(f"Retrieval error for domain '{domain}': {e}")
+            logger.exception(f"Retrieval error for domain '{domain}'.")
             return []
 
 retriever = ResearchRetriever()
@@ -212,7 +213,7 @@ class CognitiveProcessor:
             try:
                 results.append(future.result())
             except Exception as e:
-                logger.error(f"Error in API request: {e}")
+                logger.exception("Error during API request execution.")
                 st.error(f"Processing Error: {str(e)}")
 
         return self._consensus_check(results)
@@ -247,7 +248,7 @@ class CognitiveProcessor:
             logger.info("DeepSeek API request successful.")
             return response.json()
         except requests.exceptions.RequestException as e:
-            logger.error(f"DeepSeek API request failed: {e}")
+            logger.exception("DeepSeek API request failed.")
             return {"error": str(e)}
 
     def _consensus_check(self, results: List[Dict]) -> Dict:
@@ -294,12 +295,12 @@ class ResearchWorkflow:
 
     def ingest_query(self, state: AgentState) -> Dict:
         """
-        Ingests the research query and initializes the refinement counter.
+        Ingests the research query and initializes the refinement counter and history.
         """
         try:
             query = state["messages"][-1].content
-            # Initialize context with raw query and refinement counter
-            new_context = {"raw_query": query, "refine_count": 0}
+            # Initialize context with raw query, refinement counter, and an empty history list
+            new_context = {"raw_query": query, "refine_count": 0, "refinement_history": []}
             logger.info("Query ingested.")
             return {
                 "messages": [AIMessage(content="Query ingested successfully")],
@@ -307,6 +308,7 @@ class ResearchWorkflow:
                 "metadata": {"timestamp": datetime.now().isoformat()}
             }
         except Exception as e:
+            logger.exception("Error during query ingestion.")
             return self._error_state(f"Ingestion Error: {str(e)}")
 
     def retrieve_documents(self, state: AgentState) -> Dict:
@@ -319,9 +321,10 @@ class ResearchWorkflow:
             logger.info(f"Retrieved {len(docs)} documents for query.")
             return {
                 "messages": [AIMessage(content=f"Retrieved {len(docs)} documents")],
-                "context": {"documents": docs, "retrieval_time": time.time(), "refine_count": state["context"].get("refine_count", 0)}
+                "context": {"documents": docs, "retrieval_time": time.time(), "refine_count": state["context"].get("refine_count", 0), "refinement_history": state["context"].get("refinement_history", [])}
             }
         except Exception as e:
+            logger.exception("Error during document retrieval.")
             return self._error_state(f"Retrieval Error: {str(e)}")
 
     def analyze_content(self, state: AgentState) -> Dict:
@@ -334,49 +337,83 @@ class ResearchWorkflow:
             prompt = ResearchConfig.ANALYSIS_TEMPLATE.format(context=docs_text)
             response = self.processor.process_query(prompt)
             if "error" in response:
+                logger.error("DeepSeek response error during analysis.")
                 return self._error_state(response["error"])
             logger.info("Content analysis completed.")
             return {
                 "messages": [AIMessage(content=response.get('choices', [{}])[0].get('message', {}).get('content', ''))],
-                "context": {"analysis": response, "refine_count": state["context"].get("refine_count", 0)}
+                "context": {"analysis": response, "refine_count": state["context"].get("refine_count", 0), "refinement_history": state["context"].get("refinement_history", [])}
             }
         except Exception as e:
+            logger.exception("Error during content analysis.")
             return self._error_state(f"Analysis Error: {str(e)}")
 
     def validate_output(self, state: AgentState) -> Dict:
         """
         Validates the technical analysis report.
         """
-        analysis = state["messages"][-1].content
-        validation_prompt = (
-            f"Validate research analysis:\n{analysis}\n\n"
-            "Check for:\n1. Technical accuracy\n2. Citation support\n3. Logical consistency\n4. Methodological soundness\n\n"
-            "Respond with 'VALID' or 'INVALID'"
-        )
-        response = self.processor.process_query(validation_prompt)
-        logger.info("Output validation completed.")
-        return {
-            "messages": [AIMessage(content=analysis + f"\n\nValidation: {response.get('choices', [{}])[0].get('message', {}).get('content', '')}")]
-        }
+        try:
+            analysis = state["messages"][-1].content
+            validation_prompt = (
+                f"Validate research analysis:\n{analysis}\n\n"
+                "Check for:\n1. Technical accuracy\n2. Citation support\n3. Logical consistency\n4. Methodological soundness\n\n"
+                "Respond with 'VALID' or 'INVALID'"
+            )
+            response = self.processor.process_query(validation_prompt)
+            logger.info("Output validation completed.")
+            return {
+                "messages": [AIMessage(content=analysis + f"\n\nValidation: {response.get('choices', [{}])[0].get('message', {}).get('content', '')}")]
+            }
+        except Exception as e:
+            logger.exception("Error during output validation.")
+            return self._error_state(f"Validation Error: {str(e)}")
 
     def refine_results(self, state: AgentState) -> Dict:
         """
         Refines the analysis report if validation fails.
-        Increments the refinement counter to limit infinite loops.
+        Implements an innovative meta-refinement mechanism inspired by LADDER.
+        It tracks refinement history and uses a dynamic difficulty gradient.
         """
-        current_count = state["context"].get("refine_count", 0)
-        state["context"]["refine_count"] = current_count + 1
-        logger.info(f"Refinement iteration: {state['context']['refine_count']}")
-        refinement_prompt = (
-            f"Refine this analysis:\n{state['messages'][-1].content}\n\n"
-            "Improve:\n1. Technical precision\n2. Empirical grounding\n3. Theoretical coherence"
-        )
-        response = self.processor.process_query(refinement_prompt)
-        logger.info("Refinement completed.")
-        return {
-            "messages": [AIMessage(content=response.get('choices', [{}])[0].get('message', {}).get('content', ''))],
-            "context": state["context"]
-        }
+        try:
+            current_count = state["context"].get("refine_count", 0)
+            state["context"]["refine_count"] = current_count + 1
+            # Append current analysis to refinement history
+            refinement_history = state["context"].setdefault("refinement_history", [])
+            current_analysis = state["messages"][-1].content
+            refinement_history.append(current_analysis)
+            # Compute a "difficulty level" (from 3 to 0) based on refinement count
+            difficulty_level = max(0, 3 - state["context"]["refine_count"])
+            logger.info(f"Refinement iteration: {state['context']['refine_count']}, Difficulty level: {difficulty_level}")
+            
+            # If refinement count exceeds threshold, perform meta-refinement by summarizing the history
+            if state["context"]["refine_count"] >= 3:
+                meta_prompt = (
+                    "You are given the following series of refinement outputs:\n" +
+                    "\n---\n".join(refinement_history) +
+                    "\n\nSummarize the above into a final, concise, and high-quality technical analysis report. Do not introduce new ideas; just synthesize the improvements."
+                )
+                meta_response = self.processor.process_query(meta_prompt)
+                logger.info("Meta-refinement completed.")
+                return {
+                    "messages": [AIMessage(content=meta_response.get('choices', [{}])[0].get('message', {}).get('content', ''))],
+                    "context": state["context"]
+                }
+            else:
+                # Standard refinement with a dynamic difficulty prompt
+                refinement_prompt = (
+                    f"Refine this analysis (current difficulty level: {difficulty_level}):\n{current_analysis}\n\n"
+                    "Improve the following aspects:\n1. Technical precision\n2. Empirical grounding\n3. Theoretical coherence\n\n"
+                    "Use a structured difficulty gradient approach (similar to LADDER) to produce a simpler yet more accurate variant."
+                )
+                response = self.processor.process_query(refinement_prompt)
+                logger.info("Refinement completed.")
+                return {
+                    "messages": [AIMessage(content=response.get('choices', [{}])[0].get('message', {}).get('content', ''))],
+                    "context": state["context"]
+                }
+        except Exception as e:
+            logger.exception("Error during refinement.")
+            return self._error_state(f"Refinement Error: {str(e)}")
 
     def _quality_check(self, state: AgentState) -> str:
         """
@@ -502,7 +539,7 @@ class ResearchInterface:
                     self._render_event(event)
                 st.success("✅ Analysis Completed Successfully")
         except Exception as e:
-            logger.error(f"Workflow execution failed: {e}")
+            logger.exception("Workflow execution failed.")
             st.error(
                 f"""**Analysis Failed**  
 {str(e)}