Update app.py

app.py

@@ -2,61 +2,222 @@ import gradio as gr
 import os
 import time
 from cerebras.cloud.sdk import Cerebras
+import requests
+from bs4 import BeautifulSoup
+from urllib.parse import urljoin, urlparse
+from groq import Groq
+import asyncio
+import re
+import json
 
-# Set up the Cerebras client
-api_key = os.getenv("CEREBRAS_API_KEY")
-if not api_key:
+# --- Constants and API Setup ---
+CEREBRAS_API_KEY = os.getenv("CEREBRAS_API_KEY")
+if not CEREBRAS_API_KEY:
     raise ValueError("CEREBRAS_API_KEY environment variable is not set.")
 
-client = Cerebras(api_key=api_key)
+client_cerebras = Cerebras(api_key=CEREBRAS_API_KEY)
+client_groq = Groq()
 
-def chat_with_cerebras(user_input):
-    """
-    Handles interaction with the Cerebras model.
-    Sends user input and returns the model's response along with compute time and chain-of-thought reasoning.
+# --- Rate Limiting ---
+CEREBRAS_REQUESTS_PER_MINUTE = 30
+CEREBRAS_TOKENS_PER_MINUTE = 6000 # using lowest token limit for versatile model
+GROQ_REQUESTS_PER_MINUTE = 30
+GROQ_TOKENS_PER_MINUTE = 15000  # using token limit for tool-use-preview model
+
+
+cerebras_request_queue = asyncio.Queue()
+groq_request_queue = asyncio.Queue()
+
+last_cerebras_request_time = 0
+last_groq_request_time = 0
+cerebras_token_count = 0
+groq_token_count = 0
+
+# --- Model Rate Limit Info ---
+CHAT_COMPLETION_MODELS_INFO = """
+Chat Completion
+ID	Requests per Minute	Requests per Day	Tokens per Minute	Tokens per Day
+gemma-7b-it	30	14,400	15,000	500,000
+gemma2-9b-it	30	14,400	15,000	500,000
+llama-3.1-70b-versatile	30	14,400	6,000	200,000
+llama-3.1-8b-instant	30	14,400	20,000	500,000
+llama-3.2-11b-text-preview	30	7,000	7,000	500,000
+llama-3.2-11b-vision-preview	30	7,000	7,000	500,000
+llama-3.2-1b-preview	30	7,000	7,000	500,000
+llama-3.2-3b-preview	30	7,000	7,000	500,000
+llama-3.2-90b-text-preview	30	7,000	7,000	500,000
+llama-3.2-90b-vision-preview	15	3,500	7,000	250,000
+llama-3.3-70b-specdec	30	1,000	6,000	100,000
+llama-3.3-70b-versatile	30	1,000	6,000	100,000
+llama-guard-3-8b	30	14,400	15,000	500,000
+llama3-70b-8192	30	14,400	6,000	500,000
+llama3-8b-8192	30	14,400	30,000	500,000
+llama3-groq-70b-8192-tool-use-preview	30	14,400	15,000	500,000
+llama3-groq-8b-8192-tool-use-preview	30	14,400	15,000	500,000
+llava-v1.5-7b-4096-preview	30	14,400	30,000	(No limit)
+mixtral-8x7b-32768	30	14,400	5,000	500,000
+"""
+
+SPEECH_TO_TEXT_MODELS_INFO = """
+Speech To Text
+ID	Requests per Minute	Requests per Day	Audio Seconds per Hour	Audio Seconds per Day
+distil-whisper-large-v3-en	20	2,000	7,200	28,800
+whisper-large-v3	20	2,000	7,200	28,800
+whisper-large-v3-turbo	20	2,000	7,200	28,800
+"""
+
+def get_model_info():
+    return f"""
+        {CHAT_COMPLETION_MODELS_INFO}
+
+        {SPEECH_TO_TEXT_MODELS_INFO}
     """
-    # Start compute time measurement
-    start_time = time.time()
 
+
+# --- Helper Functions ---
+
+def is_valid_url(url):
     try:
-        # Create a chat stream with Cerebras
-        stream = client.chat.completions.create(
-            messages=[
-                {"role": "system", "content": "You are IntellijMind, an advanced AI designed to assist users with detailed insights, problem-solving, and chain-of-thought reasoning."},
+        result = urlparse(url)
+        return all([result.scheme, result.netloc])
+    except ValueError:
+        return False
+
+
+def fetch_webpage(url):
+    try:
+        response = requests.get(url, timeout=10)
+        response.raise_for_status()  # Raise an exception for bad status codes
+        return response.text
+    except requests.exceptions.RequestException as e:
+        return f"Error fetching URL: {e}"
+
+
+def extract_text_from_html(html):
+    soup = BeautifulSoup(html, 'html.parser')
+    text = soup.get_text(separator=' ', strip=True)
+    return text
+
+# --- Asynchronous Rate Limit Logic ---
+
+async def check_cerebras_rate_limit(num_tokens):
+    global last_cerebras_request_time
+    global cerebras_token_count
+    current_time = time.time()
+    elapsed_time = current_time - last_cerebras_request_time
+
+    if elapsed_time < 60 and cerebras_request_queue.qsize() >= CEREBRAS_REQUESTS_PER_MINUTE:
+      await asyncio.sleep(60-elapsed_time)
+
+
+    if elapsed_time < 60 and (cerebras_token_count + num_tokens) > CEREBRAS_TOKENS_PER_MINUTE :
+          time_to_wait = 60 - elapsed_time
+          await asyncio.sleep(time_to_wait)
+
+    cerebras_request_queue.put_nowait(current_time)
+    last_cerebras_request_time = time.time()
+    cerebras_token_count = num_tokens if (elapsed_time > 60) else (cerebras_token_count + num_tokens)
+
+async def check_groq_rate_limit(num_tokens):
+    global last_groq_request_time
+    global groq_token_count
+    current_time = time.time()
+    elapsed_time = current_time - last_groq_request_time
+    if elapsed_time < 60 and groq_request_queue.qsize() >= GROQ_REQUESTS_PER_MINUTE:
+      await asyncio.sleep(60 - elapsed_time)
+
+    if elapsed_time < 60 and (groq_token_count + num_tokens) > GROQ_TOKENS_PER_MINUTE :
+      time_to_wait = 60 - elapsed_time
+      await asyncio.sleep(time_to_wait)
+
+    groq_request_queue.put_nowait(current_time)
+    last_groq_request_time = time.time()
+    groq_token_count = num_tokens if (elapsed_time > 60) else (groq_token_count + num_tokens)
+
+
+# --- Chat Logic with Groq ---
+async def chat_with_groq(user_input, chat_history):
+    start_time = time.time()
+    try:
+        # Prepare chat history for the prompt
+        formatted_history = "\n".join([f"User: {msg[0]}\nAI: {msg[1]}" for msg in chat_history[-5:]])
+         # Check for web scraping command
+        if user_input.lower().startswith("scrape"):
+            parts = user_input.split(maxsplit=1)
+            if len(parts) > 1:
+                url = parts[1].strip()
+                if is_valid_url(url):
+                  html_content = fetch_webpage(url)
+                  if not html_content.startswith("Error"):
+                    webpage_text = extract_text_from_html(html_content)
+                    user_input =  f"The content from the webpage: {webpage_text}. {user_input}"
+                  else:
+                      user_input =  f"{html_content}. {user_input}"
+                else:
+                     user_input = "Invalid URL provided. " + user_input
+
+        messages = [
+                {"role": "system", "content": f"""You are IntellijMind, an advanced AI designed to assist users with detailed insights, problem-solving, and chain-of-thought reasoning. You have access to various tools to help the user, and can initiate actions when needed. Be creative and inject humor when appropriate. You can use tools to browse the web when instructed with a 'scrape' command followed by a URL.  If there is a request for model info, use the get_model_info function. Current conversation: {formatted_history} Available actions: take_action: 'scrape', parameters: url. Example action: Action: take_action, Parameters: {{"action":"scrape", "url":"https://example.com"}}  """},
                 {"role": "user", "content": user_input}
-            ],
-            model="llama-3.3-70b",
+            ]
+        if user_input.lower() == "model info":
+             response = get_model_info()
+             return response, "", f"Compute Time: {time.time() - start_time:.2f} seconds", f"Tokens used: {len(user_input.split()) + len(response.split())}"
+
+
+        num_tokens = len(user_input.split())
+        await check_groq_rate_limit(num_tokens)
+
+        completion = client_groq.chat.completions.create(
+            model="llama3-groq-70b-8192-tool-use-preview",
+            messages=messages,
+            temperature=1,
+            max_tokens=1024,
+            top_p=1,
             stream=True,
-            max_completion_tokens=1024,
-            temperature=0.2,
-            top_p=1
+            stop=None,
         )
 
-        # Collect response from the stream
         response = ""
         chain_of_thought = ""
-        for chunk in stream:
+        for chunk in completion:
             if chunk.choices[0].delta and chunk.choices[0].delta.content:
                 content = chunk.choices[0].delta.content
                 response += content
                 if "Chain of Thought:" in content:
                     chain_of_thought += content.split("Chain of Thought:", 1)[-1]
+                # Check if action needs to be taken
+                if "Action:" in content:
+                    action_match = re.search(r"Action: (\w+), Parameters: (\{.*\})", content)
+                    if action_match:
+                        action = action_match.group(1)
+                        parameters = json.loads(action_match.group(2))
+                        if action == "take_action":
+                           if parameters.get("action") == "scrape":
+                                url = parameters.get("url")
+                                if is_valid_url(url):
+                                  html_content = fetch_webpage(url)
+                                  if not html_content.startswith("Error"):
+                                    webpage_text = extract_text_from_html(html_content)
+                                    response += f"\nWebpage Content: {webpage_text}\n"
+                                  else:
+                                     response += f"\nError scraping webpage: {html_content}\n"
+                                else:
+                                   response +=  "\nInvalid URL provided.\n"
 
-        # End compute time measurement
-        compute_time = time.time() - start_time
 
-        # Simulate token usage tracking (placeholder for real implementation)
+        compute_time = time.time() - start_time
         token_usage = len(user_input.split()) + len(response.split())
-
         return response, chain_of_thought, f"Compute Time: {compute_time:.2f} seconds", f"Tokens used: {token_usage}"
 
     except Exception as e:
         return "Error: Unable to process your request.", "", str(e), ""
 
-# Gradio interface
+
+# --- Gradio Interface ---
 def gradio_ui():
     with gr.Blocks() as demo:
-        gr.Markdown("""# 🚀 IntellijMind: The Future of AI Chatbots\nExperience the most advanced chatbot for deep insights, chain-of-thought reasoning, and unmatched clarity!""")
+        gr.Markdown("""# 🚀 IntellijMind: The Crazy Agent Chatbot\nExperience the most advanced chatbot for deep insights, chain-of-thought reasoning, and unmatched clarity! Get ready for some proactive action!""")
 
         with gr.Row():
             with gr.Column(scale=6):
@@ -68,15 +229,19 @@ def gradio_ui():
 
         user_input = gr.Textbox(label="Type your message", placeholder="Ask me anything...", lines=2)
 
+
         with gr.Row():
             send_button = gr.Button("Send", variant="primary")
             clear_button = gr.Button("Clear Chat")
             export_button = gr.Button("Export Chat History")
 
-        def handle_chat(chat_history, user_input):
+        async def handle_chat(chat_history, user_input):
             if not user_input.strip():
                 return chat_history, "", "", "", "Please enter a valid message."
-            ai_response, chain_of_thought, compute_info, token_usage = chat_with_cerebras(user_input)
+
+            ai_response, chain_of_thought, compute_info, token_usage = await chat_with_groq(user_input, chat_history)
+
+
             chat_history.append((user_input, ai_response))
             return chat_history, chain_of_thought, compute_info, token_usage
 
@@ -98,10 +263,10 @@ def gradio_ui():
 
         user_input.submit(handle_chat, inputs=[chat_history, user_input], outputs=[chat_history, chain_of_thought_display, compute_time, token_usage_display])
 
-        gr.Markdown("""---\n### 🌟 Features:\n- **Advanced Reasoning**: Chain-of-thought explanations for complex queries.\n- **Real-Time Performance Metrics**: Measure response compute time instantly.\n- **Token Usage Tracking**: Monitor token usage per response for transparency.\n- **Export Chat History**: Save your conversation as a text file for future reference.\n- **User-Friendly Design**: Intuitive chatbot interface with powerful features.\n- **Insightful Chain of Thought**: See the reasoning process behind AI decisions.\n- **Submit on Enter**: Seamless interaction with keyboard support.\n""")
+        gr.Markdown("""---\n### 🌟 Features:\n- **Advanced Reasoning**: Chain-of-thought explanations for complex queries.\n- **Proactive Actions**: The agent will take actions without being explicitly asked.\n- **Web Scraping**: The agent will use the scrape command if needed\n- **Humor and Creativity**: Enjoy a more engaging and creative experience.\n- **Real-Time Performance Metrics**: Measure response compute time instantly.\n- **Token Usage Tracking**: Monitor token usage per response for transparency.\n- **Export Chat History**: Save your conversation as a text file for future reference.\n- **User-Friendly Design**: Intuitive chatbot interface with powerful features.\n- **Insightful Chain of Thought**: See the reasoning process behind AI decisions.\n- **Submit on Enter**: Seamless interaction with keyboard support.\n""")
 
     return demo
 
 # Run the Gradio app
 demo = gradio_ui()
-demo.launch()
+demo.launch()