Update app.py

app.py

@@ -1,5 +1,5 @@
 # ------------------------------
-# UniversalResearch AI System + LADDER (Tufa Labs)
+# UniversalResearch AI with LADDER (OpenAI Integration)
 # ------------------------------
 import logging
 import os
@@ -16,7 +16,7 @@ import chromadb
 import requests
 import streamlit as st
 
-# LangChain and LangGraph imports
+# LangChain & LangGraph imports
 from langchain_openai import OpenAIEmbeddings
 from langchain_community.vectorstores import Chroma
 from langchain_core.messages import HumanMessage, AIMessage, ToolMessage
@@ -27,7 +27,7 @@ from langgraph.graph.message import add_messages
 from typing_extensions import TypedDict, Annotated
 from langchain.tools.retriever import create_retriever_tool
 
-# Increase Python's recursion limit if needed
+# Increase Python's recursion limit at the start (if needed)
 sys.setrecursionlimit(10000)
 
 # ------------------------------
@@ -45,8 +45,9 @@ logger = logging.getLogger(__name__)
 class AgentState(TypedDict):
     """
     Stores the messages and context for each step in the workflow.
-    'messages' contain the conversation so far,
-    'context' can hold domain-specific data, 'metadata' for additional info.
+    'messages': conversation so far
+    'context': domain-specific data (docs, counters)
+    'metadata': any additional info (timestamps, status)
     """
     messages: Annotated[Sequence[AIMessage | HumanMessage | ToolMessage], add_messages]
     context: Dict[str, Any]
@@ -57,47 +58,45 @@ class AgentState(TypedDict):
 # ------------------------------
 class ResearchConfig:
     """
-    Universal configuration for the research system, referencing Tufa Labs' LADDER approach.
+    Universal config for the advanced AI system with Tufa Labs' LADDER approach,
+    using OpenAI for both embeddings and completions.
 
-    Make sure to set DEEPSEEK_API_KEY in your environment or HF Space secrets
-    to enable the external LLM calls.
+    Make sure to set OPENAI_API_KEY in your environment or HF Space secrets.
     """
-    DEEPSEEK_API_KEY = os.environ.get("DEEPSEEK_API_KEY")
+    OPENAI_API_KEY = os.environ.get("OPENAI_API_KEY")  # Must match your HF secret name
     CHROMA_PATH = "chroma_db"
     CHUNK_SIZE = 512
     CHUNK_OVERLAP = 64
     MAX_CONCURRENT_REQUESTS = 5
     EMBEDDING_DIMENSIONS = 1536
 
-    # Example map for featured research docs
+    # Example map for featured documents
     DOCUMENT_MAP = {
         "Sample Research Document 1": "Topic A Overview",
         "Sample Research Document 2": "Topic B Analysis",
         "Sample Research Document 3": "Topic C Innovations"
     }
 
-    # Template referencing a general approach for analyzing research documents
+    # Analysis template referencing LADDER's approach
     ANALYSIS_TEMPLATE = (
         "Analyze the following research documents with scientific rigor:\n{context}\n\n"
-        "Using the LADDER approach from Tufa Labs, the model should:\n"
-        "1. Break down the problem into simpler subproblems.\n"
+        "Use Tufa Labs’ LADDER method to:\n"
+        "1. Break down complex problems into subproblems.\n"
         "2. Iteratively refine the solution.\n"
-        "3. Provide thorough analysis, including:\n"
+        "3. Provide analysis including:\n"
         "   a. Key Contributions\n"
         "   b. Novel Methodologies\n"
         "   c. Empirical Results (with metrics)\n"
         "   d. Potential Applications\n"
         "   e. Limitations & Future Directions\n\n"
-        "Format your response in Markdown with LaTeX mathematical notation where applicable."
+        "Format your response in Markdown with LaTeX where applicable."
     )
 
 # Early check for missing API key
-if not ResearchConfig.DEEPSEEK_API_KEY:
+if not ResearchConfig.OPENAI_API_KEY:
     st.error(
-        """**Research Portal Configuration Required**  
-1. Obtain your DeepSeek API key from [platform.deepseek.com](https://platform.deepseek.com/)  
-2. Set the secret: `DEEPSEEK_API_KEY` in your Space settings  
-3. Rebuild your deployment."""
+        """**OpenAI API Key Not Found**  
+Please set `OPENAI_API_KEY` in your Space secrets and rebuild the Space."""
     )
     st.stop()
 
@@ -106,8 +105,8 @@ if not ResearchConfig.DEEPSEEK_API_KEY:
 # ------------------------------
 class UniversalDocumentManager:
     """
-    Manages creation of document collections for any research domain.
-    Uses OpenAI embeddings for vector-based semantic search.
+    Manages creation of document collections for any research domain,
+    using OpenAI embeddings for semantic search.
     """
     def __init__(self) -> None:
         try:
@@ -117,6 +116,7 @@ class UniversalDocumentManager:
             logger.error(f"Error initializing PersistentClient: {e}")
             self.client = chromadb.Client()  # Fallback to in-memory client
 
+        # Configure embeddings from openai
         self.embeddings = OpenAIEmbeddings(
             model="text-embedding-3-large",
             dimensions=ResearchConfig.EMBEDDING_DIMENSIONS
@@ -124,7 +124,7 @@ class UniversalDocumentManager:
 
     def create_collection(self, documents: List[str], collection_name: str) -> Chroma:
         """
-        Splits documents into manageable chunks and stores them in a Chroma collection.
+        Splits documents into chunks and stores them in a Chroma collection.
         """
         splitter = RecursiveCharacterTextSplitter(
             chunk_size=ResearchConfig.CHUNK_SIZE,
@@ -133,7 +133,7 @@ class UniversalDocumentManager:
         )
         try:
             docs = splitter.create_documents(documents)
-            logger.info(f"Created {len(docs)} document chunks for collection '{collection_name}'.")
+            logger.info(f"Created {len(docs)} doc chunks for collection '{collection_name}'.")
         except Exception as e:
             logger.error(f"Error splitting documents: {e}")
             raise e
@@ -148,11 +148,11 @@ class UniversalDocumentManager:
 
     def _document_id(self, content: str) -> str:
         """
-        Generates a unique document ID using SHA256 + timestamp.
+        Generates a unique ID using SHA256 + timestamp.
         """
         return f"{hashlib.sha256(content.encode()).hexdigest()[:16]}-{int(time.time())}"
 
-# Create example collections (can be replaced with domain-specific docs)
+# Example collections (replace with your own)
 udm = UniversalDocumentManager()
 research_docs = udm.create_collection([
     "Research Report: Novel AI Techniques in Renewable Energy",
@@ -171,8 +171,8 @@ development_docs = udm.create_collection([
 # ------------------------------
 class ResearchRetriever:
     """
-    Provides retrieval methods for multiple domains (research, development, etc.).
-    Uses MMR (Maximal Marginal Relevance) or similarity-based retrieval from Chroma.
+    Provides retrieval methods for multiple domains (e.g., research, development).
+    Uses MMR or similarity-based retrieval from Chroma.
     """
     def __init__(self) -> None:
         try:
@@ -200,7 +200,7 @@ class ResearchRetriever:
             elif domain == "development":
                 return self.development_retriever.invoke(query)
             else:
-                logger.warning(f"Domain '{domain}' not recognized. Defaulting to research.")
+                logger.warning(f"Domain '{domain}' not recognized. Defaulting to 'research'.")
                 return self.research_retriever.invoke(query)
         except Exception as e:
             logger.error(f"Retrieval error for domain '{domain}': {e}")
@@ -213,8 +213,8 @@ retriever = ResearchRetriever()
 # ------------------------------
 class CognitiveProcessor:
     """
-    Sends parallel requests to the DeepSeek API to reduce failures.
-    Implements a consensus mechanism to pick the most comprehensive response.
+    Executes requests to the OpenAI Chat Completions endpoint in parallel,
+    then consolidates the results using a consensus mechanism (picks the longest).
     """
     def __init__(self) -> None:
         self.executor = ThreadPoolExecutor(max_workers=ResearchConfig.MAX_CONCURRENT_REQUESTS)
@@ -222,7 +222,7 @@ class CognitiveProcessor:
 
     def process_query(self, prompt: str) -> Dict:
         """
-        Processes a query by sending multiple parallel requests (triple redundancy).
+        Sends multiple parallel requests (triple redundancy) to OpenAI's ChatCompletion.
         """
         futures = []
         for _ in range(3):
@@ -240,61 +240,63 @@ class CognitiveProcessor:
 
     def _execute_api_request(self, prompt: str) -> Dict:
         """
-        Executes a single request to the DeepSeek endpoint.
+        Executes a single request to OpenAI's ChatCompletion endpoint.
         """
+        # Use your OPENAI_API_KEY
         headers = {
-            "Authorization": f"Bearer {ResearchConfig.DEEPSEEK_API_KEY}",
-            "Content-Type": "application/json",
-            "X-Research-Session": self.session_id
+            "Authorization": f"Bearer {ResearchConfig.OPENAI_API_KEY}",
+            "Content-Type": "application/json"
         }
         payload = {
-            "model": "deepseek-chat",
-            "messages": [{
-                "role": "user",
-                "content": f"Respond as a Senior Researcher:\n{prompt}"
-            }],
+            "model": "gpt-3.5-turbo",  # or "gpt-4", depending on your account
+            "messages": [
+                {
+                    "role": "user",
+                    "content": prompt
+                }
+            ],
             "temperature": 0.7,
             "max_tokens": 1500,
             "top_p": 0.9
         }
         try:
             response = requests.post(
-                "https://api.deepseek.com/v1/chat/completions",
+                "https://api.openai.com/v1/chat/completions",
                 headers=headers,
                 json=payload,
                 timeout=45
             )
             response.raise_for_status()
-            logger.info("DeepSeek API request successful.")
+            logger.info("OpenAI ChatCompletion request successful.")
             return response.json()
         except requests.exceptions.RequestException as e:
-            logger.error(f"DeepSeek API request failed: {e}")
+            logger.error(f"OpenAI request failed: {e}")
             return {"error": str(e)}
 
     def _consensus_check(self, results: List[Dict]) -> Dict:
         """
-        Chooses the best response by comparing the length of the message content.
+        Chooses the 'best' response by comparing content lengths, discarding errors.
         """
-        valid_results = [r for r in results if "error" not in r]
-        if not valid_results:
+        valid = [r for r in results if "error" not in r]
+        if not valid:
             logger.error("All API requests failed.")
             return {"error": "All API requests failed"}
-        return max(valid_results, key=lambda x: len(x.get('choices', [{}])[0].get('message', {}).get('content', '')))
+        return max(valid, key=lambda x: len(x.get('choices', [{}])[0].get('message', {}).get('content', '')))
 
 # ------------------------------
-# Research Workflow Engine (LADDER Integration)
+# Research Workflow Engine (Tufa Labs' LADDER)
 # ------------------------------
 class ResearchWorkflow:
     """
-    Defines a multi-step workflow using LangGraph with Tufa Labs' LADDER approach:
+    Defines a multi-step workflow using LangGraph with Tufa Labs’ LADDER approach:
     1. Ingest Query
     2. Retrieve Documents
     3. Analyze Content
     4. Validate Output
-    5. Refine (Recursive Self-Learning + TTRL)
+    5. Refine (Recursive Self-Learning)
     
-    The 'refine_results' node applies LADDER’s idea of iteratively
-    breaking down problems and re-solving them with no external data.
+    The refine step uses iterative subproblem breakdown, 
+    potentially combined with test-time reinforcement.
     """
     def __init__(self) -> None:
         self.processor = CognitiveProcessor()
@@ -303,14 +305,14 @@ class ResearchWorkflow:
         self.app = self.workflow.compile()
 
     def _build_workflow(self) -> None:
-        # Define workflow nodes
+        # Node definitions
         self.workflow.add_node("ingest", self.ingest_query)
         self.workflow.add_node("retrieve", self.retrieve_documents)
         self.workflow.add_node("analyze", self.analyze_content)
         self.workflow.add_node("validate", self.validate_output)
         self.workflow.add_node("refine", self.refine_results)
 
-        # Entry point and transitions
+        # Graph edges
         self.workflow.set_entry_point("ingest")
         self.workflow.add_edge("ingest", "retrieve")
         self.workflow.add_edge("retrieve", "analyze")
@@ -324,7 +326,7 @@ class ResearchWorkflow:
 
     def ingest_query(self, state: AgentState) -> Dict:
         """
-        Ingests the research query and initializes the LADDER-based refinement counter.
+        Ingest the user query and initialize the refine counter for LADDER recursion.
         """
         try:
             query = state["messages"][-1].content
@@ -336,12 +338,11 @@ class ResearchWorkflow:
                 "metadata": {"timestamp": datetime.now().isoformat()}
             }
         except Exception as e:
-            return self._error_state(f"Ingestion Error: {str(e)}")
+            return self._error_state(f"Ingestion Error: {e}")
 
     def retrieve_documents(self, state: AgentState) -> Dict:
         """
-        Retrieves relevant documents based on the query. 
-        The system can handle any domain (math, code generation, theorem proving, etc.).
+        Retrieves relevant documents from the specified domain (default: research).
         """
         try:
             query = state["context"]["raw_query"]
@@ -356,14 +357,12 @@ class ResearchWorkflow:
                 }
             }
         except Exception as e:
-            return self._error_state(f"Retrieval Error: {str(e)}")
+            return self._error_state(f"Retrieval Error: {e}")
 
     def analyze_content(self, state: AgentState) -> Dict:
         """
-        Analyzes the retrieved documents using Tufa Labs' LADDER principles:
-        - Break down the documents,
-        - Provide structured analysis,
-        - Return a refined solution.
+        Uses the LADDER approach to break down and analyze documents,
+        returning a structured research analysis.
         """
         try:
             docs = state["context"].get("documents", [])
@@ -372,12 +371,10 @@ class ResearchWorkflow:
             response = self.processor.process_query(prompt)
             if "error" in response:
                 return self._error_state(response["error"])
-            logger.info("Content analysis completed.")
+            logger.info("Analysis completed.")
             return {
                 "messages": [
-                    AIMessage(
-                        content=response.get('choices', [{}])[0].get('message', {}).get('content', '')
-                    )
+                    AIMessage(content=response.get('choices', [{}])[0].get('message', {}).get('content', ''))
                 ],
                 "context": {
                     "analysis": response,
@@ -385,12 +382,12 @@ class ResearchWorkflow:
                 }
             }
         except Exception as e:
-            return self._error_state(f"Analysis Error: {str(e)}")
+            return self._error_state(f"Analysis Error: {e}")
 
     def validate_output(self, state: AgentState) -> Dict:
         """
-        Validates the analysis. If invalid, the system can refine the solution
-        (potentially multiple times) using LADDER’s iterative approach.
+        Validates the analysis. If invalid, the system can refine 
+        using Tufa Labs’ LADDER approach.
         """
         analysis = state["messages"][-1].content
         validation_prompt = (
@@ -399,7 +396,7 @@ class ResearchWorkflow:
             "Respond with 'VALID' or 'INVALID'."
         )
         response = self.processor.process_query(validation_prompt)
-        logger.info("Output validation completed.")
+        logger.info("Validation completed.")
         return {
             "messages": [
                 AIMessage(
@@ -410,22 +407,18 @@ class ResearchWorkflow:
 
     def refine_results(self, state: AgentState) -> Dict:
         """
-        Applies Tufa Labs' LADDER principle:
-        - Recursively break down the problem,
-        - Re-solve with no external data,
-        - Potentially leverage TTRL for dynamic updates.
-
-        This method increments a refinement counter to avoid infinite recursion.
+        LADDER refinement: break down subproblems, re-solve them 
+        with no external data, potentially using TTRL for dynamic updates.
         """
         current_count = state["context"].get("refine_count", 0)
         state["context"]["refine_count"] = current_count + 1
         logger.info(f"LADDER refinement iteration: {state['context']['refine_count']}")
 
         refinement_prompt = (
-            f"Refine this analysis using LADDER's self-improvement approach:\n"
+            "Refine this analysis with LADDER’s self-improvement approach:\n"
             f"{state['messages'][-1].content}\n\n"
-            "Focus on breaking down complex points further and re-solving them.\n"
-            "Enhance technical precision, empirical grounding, and theoretical coherence."
+            "Break down complex points further, re-solve them, and enhance:\n"
+            "- Technical precision\n- Empirical grounding\n- Theoretical coherence"
         )
         response = self.processor.process_query(refinement_prompt)
         logger.info("Refinement completed.")
@@ -440,19 +433,20 @@ class ResearchWorkflow:
 
     def _quality_check(self, state: AgentState) -> str:
         """
-        Determines if the analysis is 'valid' or 'invalid'. 
-        If refine_count exceeds 3, forcibly accept the result to prevent infinite loops.
+        Checks if the analysis is valid. If the refine_count >= 3,
+        forcibly accept to avoid infinite loops.
         """
         refine_count = state["context"].get("refine_count", 0)
         if refine_count >= 3:
-            logger.warning("Refinement limit reached. Forcing valid outcome to prevent infinite recursion.")
+            logger.warning("Refinement limit reached. Forcing valid outcome.")
             return "valid"
+
         content = state["messages"][-1].content
         return "valid" if "VALID" in content else "invalid"
 
     def _error_state(self, message: str) -> Dict:
         """
-        Returns an error state if any node fails. 
+        Returns an error state if any node fails.
         """
         logger.error(message)
         return {
@@ -462,12 +456,12 @@ class ResearchWorkflow:
         }
 
 # ------------------------------
-# Research Interface (Streamlit UI)
+# Streamlit UI
 # ------------------------------
 class ResearchInterface:
     """
-    Provides a Streamlit-based interface for the UniversalResearch AI with LADDER. 
-    The system is domain-agnostic, handling math, code generation, theorem proving, etc.
+    Provides a Streamlit-based interface for the UniversalResearch AI 
+    with Tufa Labs' LADDER approach, using OpenAI for both embeddings & completions.
     """
     def __init__(self) -> None:
         self.workflow = ResearchWorkflow()
@@ -475,7 +469,7 @@ class ResearchInterface:
 
     def _initialize_interface(self) -> None:
         st.set_page_config(
-            page_title="UniversalResearch AI (LADDER)",
+            page_title="UniversalResearch AI (OpenAI + LADDER)",
             layout="wide",
             initial_sidebar_state="expanded"
         )
@@ -541,12 +535,12 @@ class ResearchInterface:
     def _build_main_interface(self) -> None:
         st.title("🧠 UniversalResearch AI")
         st.write(
-            "Leverage the power LADDER approach for recursive self-improvement. "
+            "This system uses OpenAI for embeddings & completions"
         )
         query = st.text_area(
             "Research Query:",
             height=200,
-            placeholder="Enter a research question, from math to code generation..."
+            placeholder="Enter a research question (e.g., advanced math, code tasks, etc.)..."
         )
         if st.button("Execute Analysis", type="primary"):
             self._execute_analysis(query)
@@ -554,7 +548,7 @@ class ResearchInterface:
     def _execute_analysis(self, query: str) -> None:
         try:
             with st.spinner("Initializing LADDER-based Analysis..."):
-                # The recursion_limit config ensures we can handle multiple refine iterations
+                # The recursion_limit ensures multiple refine iterations are possible
                 results = self.workflow.app.stream({
                     "messages": [HumanMessage(content=query)],
                     "context": {},
@@ -571,12 +565,13 @@ class ResearchInterface:
 Potential issues:
 - Complex query structure
 - Document correlation failure
+- Rate limits or invalid API key
 - Temporal processing constraints"""
             )
 
     def _render_event(self, event: Dict) -> None:
         """
-        Renders each event in the Streamlit UI.
+        Renders each event in the Streamlit UI, from ingestion to validation/refinement.
         """
         if 'ingest' in event:
             with st.container():
@@ -600,7 +595,7 @@ Potential issues:
                 if "VALID" in content:
                     st.success("✅ Validation Passed")
                     with st.expander("View Validated Analysis", expanded=True):
-                        # Remove "Validation: ..." for a cleaner final result
+                        # Hide "Validation: ..." from final output
                         st.markdown(content.split("Validation:")[0])
                 else:
                     st.warning("⚠️ Validation Issues Detected")