Update app.py

app.py

@@ -233,6 +233,9 @@ def generate_chunked_response(model, prompt, max_tokens=1000, max_chunks=5):
             full_response += chunk
         except Exception as e:
             print(f"Error in generate_chunked_response: {e}")
+            if "Input validation error" in str(e):
+                # If we hit the token limit, return what we have so far
+                return full_response if full_response else "The input was too long to process. Please try a shorter query."
             break
     return full_response.strip()
 
@@ -346,8 +349,8 @@ def ask_question(question, temperature, top_p, repetition_penalty, web_search, c
         database = None
 
     max_attempts = 5
-    context_reduction_factor = 0.7
-    max_estimated_tokens = 30000  # Leave some room for the model's response
+    context_reduction_factor = 0.5  # More aggressive reduction
+    max_estimated_tokens = 25000  # Further reduced to leave more room for response
 
     if web_search:
         contextualized_question, topics, entity_tracker, instructions = chatbot.process_question(question)
@@ -358,7 +361,7 @@ def ask_question(question, temperature, top_p, repetition_penalty, web_search, c
 
         for attempt in range(max_attempts):
             try:
-                web_docs = [Document(page_content=result["text"], metadata={"source": result["link"]}) for result in search_results if result["text"]]
+                web_docs = [Document(page_content=result["text"][:1000], metadata={"source": result["link"]}) for result in search_results if result["text"]]  # Limit each result to 1000 characters
 
                 if database is None:
                     database = FAISS.from_documents(web_docs, embed)
@@ -372,20 +375,16 @@ def ask_question(question, temperature, top_p, repetition_penalty, web_search, c
                 instruction_prompt = f"User Instructions: {instructions}\n" if instructions else ""
 
                 prompt_template = f"""
-                Answer the question based on the following web search results, conversation context, entity information, and user instructions:
-                Web Search Results:
-                {{context}}
-                Conversation Context: {{conv_context}}
-                Current Question: {{question}}
+                Answer based on: Web Results: {{context}}
+                Context: {{conv_context}}
+                Question: {{question}}
                 Topics: {{topics}}
-                Entity Information: {{entities}}
+                Entities: {{entities}}
                 {instruction_prompt}
-                Provide a concise and relevant answer to the question.
                 """
 
                 prompt_val = ChatPromptTemplate.from_template(prompt_template)
                 
-                # Start with full context and progressively reduce if necessary
                 current_context = context_str
                 current_conv_context = chatbot.get_context()
                 current_topics = topics
@@ -393,14 +392,13 @@ def ask_question(question, temperature, top_p, repetition_penalty, web_search, c
 
                 while True:
                     formatted_prompt = prompt_val.format(
-                        context=current_context,
-                        conv_context=current_conv_context,
+                        context=current_context[:3000],  # Limit context to 3000 characters
+                        conv_context=current_conv_context[:500],  # Limit conversation context to 500 characters
                         question=question,
-                        topics=", ".join(current_topics),
-                        entities=json.dumps(current_entities)
+                        topics=", ".join(current_topics[:5]),  # Limit to 5 topics
+                        entities=json.dumps({k: v[:2] for k, v in current_entities.items()})  # Limit to 2 entities per type
                     )
 
-                    # Estimate token count
                     estimated_tokens = estimate_tokens(formatted_prompt)
                     
                     if estimated_tokens <= max_estimated_tokens: