Update app.py

app.py

@@ -1,24 +1,27 @@
 import gradio as gr
 from huggingface_hub import InferenceClient
+from transformers import AutoTokenizer
 
-client = InferenceClient("Qwen/Qwen2.5-Coder-32B-Instruct")
+client = InferenceClient("Qwen/Qwen2.5-72B-Instruct")
+tokenizer = AutoTokenizer.from_pretrained("Qwen/Qwen2.5-Coder-32B-Instruct")
 
 # Global variables
 uploaded_file_content = ""
-MAX_CHARS = 50000  # Maximum characters to send to API
 
-def truncate_text(text, max_length=MAX_CHARS):
-    """Truncate text to max_length while trying to preserve complete sentences."""
-    if len(text) <= max_length:
+def truncate_text_by_tokens(text, max_tokens=2048):
+    """Truncate text to max_tokens while trying to preserve complete sentences."""
+    # Tokenize the input text
+    tokens = tokenizer.encode(text)
+    
+    # If the text is already within the token limit, return as is
+    if len(tokens) <= max_tokens:
         return text
     
-    # Try to find the last period before max_length
-    last_period = text[:max_length].rfind('.')
-    if last_period != -1:
-        return text[:last_period + 1]
+    # Otherwise, truncate the token list and decode it back to text
+    truncated_tokens = tokens[:max_tokens]
+    truncated_text = tokenizer.decode(truncated_tokens, skip_special_tokens=True)
     
-    # If no period found, just truncate at max_length
-    return text[:max_length] + "..."
+    return truncated_text
 
 def handle_file_upload(file_obj):
     global uploaded_file_content
@@ -34,15 +37,14 @@ def handle_file_upload(file_obj):
         
         # Store full content but truncate for preview
         uploaded_file_content = file_content
-        truncated = truncate_text(file_content, MAX_CHARS)  # Store full but preview truncated
-        preview = truncated[:200] + "..."
+        preview = truncate_text_by_tokens(file_content, max_tokens=100)[:200] + "..."  # Preview truncated content
         
-        total_chars = len(file_content)
-        usable_chars = len(truncated)
+        total_tokens = len(tokenizer.encode(file_content))
+        usable_tokens = len(tokenizer.encode(preview))
         
         return f"""File uploaded successfully! 
-Total length: {total_chars} characters
-Usable length for AI: {usable_chars} characters (due to API limits)
+Total length: {total_tokens} tokens
+Usable length for AI: {usable_tokens} tokens (due to API limits)
 Preview of beginning:
 {preview}"""
     
@@ -59,8 +61,8 @@ def respond(
 ):
     global uploaded_file_content
     
-    # Truncate file content if needed
-    truncated_content = truncate_text(uploaded_file_content) if uploaded_file_content else ""
+    # Truncate file content if needed based on token limit
+    truncated_content = truncate_text_by_tokens(uploaded_file_content, max_tokens) if uploaded_file_content else ""
     
     # Format the current message to include truncated file content
     current_message = message
@@ -169,4 +171,4 @@ with demo:
     )
 
 if __name__ == "__main__":
-    demo.launch()
+    demo.launch()