multimodal ai chatbot

.gitignore

@@ -0,0 +1,2 @@
+multiaichat
+.env

app.py

@@ -0,0 +1,83 @@
+import gradio as gr
+from datetime import datetime
+from modules.audio_processing import transcribe_audio
+from modules.image_processing import encode_image_to_base64
+from modules.text_processing import get_user_query
+from modules.gemini_response import generate_medical_response
+from modules.reminder_scheduler import add_reminder
+
+# Chatbot logic
+def chatbot_fn(user_message, history, audio_file=None, image_file=None, tone="Friendly and Simple"):
+    transcribed_text = None
+    if audio_file:
+        transcribed_text = transcribe_audio(audio_file)
+
+    query = get_user_query(user_message, transcribed_text)
+    image_base64 = encode_image_to_base64(image_file) if image_file else None
+
+    tone_instruction = f"(Respond in a '{tone}' tone)\n"
+    full_query = tone_instruction + query
+
+    messages = history or []
+    messages.append({"role": "user", "content": full_query})
+
+    response = generate_medical_response(full_query, image_base64=image_base64, history=messages)
+
+    return response
+
+chatbot_ui = gr.ChatInterface(
+    fn=chatbot_fn,
+    additional_inputs=[
+        gr.Audio(sources=["microphone"], type="filepath", label="🎤 Speak (optional)"),
+        gr.Image(type="filepath", label="🖼️ Upload Medical Image (optional)"),
+        gr.Dropdown(["Friendly and Simple", "Detailed and Clinical", "Explain Like I'm 5"],
+                    label="🗣️ Response Tone", value="Friendly and Simple")
+    ],
+    title="🩺 Medical AI Chatbot",
+    description="Ask medical questions using text, voice, or images. The AI will respond like a helpful doctor.",
+    theme="soft",
+    type="messages"
+)
+
+# Reminder logic
+def submit_reminder(subject, description, time_str, email, confirm):
+    if not confirm:
+        return "❌ Please check the confirmation box before submitting."
+    if not subject or not time_str or not email:
+        return "❌ Please fill in Subject, Time, and Email."
+
+    try:
+        parsed_time = datetime.strptime(time_str.strip(), "%I:%M %p")
+        #print(f"⏰ Parsed time input: {parsed_time}") 
+        reminder_time = datetime.combine(datetime.today(), parsed_time.time())
+    except ValueError:
+        return "❌ Use HH:MM AM/PM format (e.g. 09:00 PM)"
+
+    full_description = f"{subject} — {description}" if description else subject
+    add_reminder(email=email, med=full_description, time_obj=reminder_time)
+
+    return f"✅ Reminder set for '{subject}' at {reminder_time.strftime('%I:%M %p')} to {email}."
+
+with gr.Blocks() as reminder_ui:
+    gr.Markdown("### ⏰ Set Medication Reminder")
+
+    with gr.Row():
+        subject_input = gr.Textbox(label="🧪 Subject", placeholder="Insulin Injection")
+        time_input = gr.Textbox(label="🕒 Time (e.g. 09:00 PM)", placeholder="HH:MM AM/PM")
+
+    with gr.Row():
+        email_input = gr.Textbox(label="📧 Email", placeholder="[email protected]")
+        confirm_checkbox = gr.Checkbox(label="✅ Confirm to receive email reminder")
+
+    description_input = gr.Textbox(label="📝 Description (optional)", placeholder="Take after dinner")
+
+    send_btn = gr.Button("📩 Set Reminder")
+    output_text = gr.Textbox(label="Status")
+
+    send_btn.click(
+        fn=submit_reminder,
+        inputs=[subject_input, description_input, time_input, email_input, confirm_checkbox],
+        outputs=output_text
+    )
+
+gr.TabbedInterface([chatbot_ui, reminder_ui], ["💬 Medical Chatbot", "⏰ Medication Reminder"]).launch(debug=True)

app_1.py

@@ -0,0 +1,48 @@
+import gradio as gr
+from modules.audio_processing import transcribe_audio
+from modules.image_processing import encode_image_to_base64
+from modules.text_processing import get_user_query
+#from modules.ai_response import generate_medical_response
+from modules.gemini_response import generate_medical_response
+
+def process_inputs(text_input, audio_file, image_file, history):
+    transcribed_text = None
+    if audio_file:
+        transcribed_text = transcribe_audio(audio_file)
+
+    query = get_user_query(text_input, transcribed_text)
+    image_base64 = encode_image_to_base64(image_file) if image_file else None
+
+    if history is None:
+        history = []
+
+    # Add user message to history
+    history.append({"role": "user", "content": query})
+
+    # Get doctor response
+    doctor_response = generate_medical_response(query, image_base64=image_base64, history=history)
+
+    # Add doctor response to history
+    history.append({"role": "assistant", "content": doctor_response})
+
+    return transcribed_text or "", doctor_response, history
+
+iface = gr.Interface(
+    fn=process_inputs,
+    inputs=[
+        gr.Textbox(label="Type your medical question (optional)"),
+        gr.Audio(sources=["microphone"], type="filepath", label="Speak your question (optional)"),
+        gr.Image(type="filepath", label="Upload a related medical image (optional)"),
+        gr.State([])  # <-- Chat history
+    ],
+    outputs=[
+        gr.Textbox(label="Transcribed Audio"),
+        gr.Textbox(label="Doctor's Detailed Response"),
+        gr.State()  # <-- Return updated history
+    ],
+    title="Multimodal Medical AI Chatbot",
+    description="Ask about diseases or chronic illnesses using text, audio, or images."
+)
+
+if __name__ == "__main__":
+    iface.launch(debug=True)

modules/ai_response.py

@@ -0,0 +1,36 @@
+import os
+from dotenv import load_dotenv
+from groq import Groq
+
+load_dotenv()
+GROQ_API_KEY = os.getenv("GROQ_API_KEY")
+
+def generate_medical_response(query, image_base64=None, history=None, model="meta-llama/llama-4-scout-17b-16e-instruct"):
+    client = Groq(api_key=GROQ_API_KEY)
+
+    prompt_path = os.path.join(os.path.dirname(__file__), "../prompts/system_prompt.txt")
+    with open(prompt_path, "r") as f:
+        system_prompt = f.read().strip()
+
+    messages = [{"role": "system", "content": system_prompt}]
+
+    # Add past conversation turns (if any)
+    if history:
+        messages.extend(history[-5:])  # Use last 5 exchanges max
+
+    # Append current query + image
+    user_content = [{"type": "text", "text": query}]
+    if image_base64:
+        user_content.append({
+            "type": "image_url",
+            "image_url": {"url": f"data:image/jpeg;base64,{image_base64}"}
+        })
+
+    messages.append({"role": "user", "content": user_content})
+
+    response = client.chat.completions.create(
+        model=model,
+        messages=messages
+    )
+
+    return response.choices[0].message.content

modules/audio_processing.py

@@ -0,0 +1,16 @@
+import os
+from groq import Groq
+from dotenv import load_dotenv
+
+load_dotenv()
+GROQ_API_KEY = os.getenv("GROQ_API_KEY")
+
+def transcribe_audio(audio_filepath, stt_model='whisper-large-v3'):
+    client = Groq(api_key=GROQ_API_KEY)
+    with open(audio_filepath, "rb") as audio_file:
+        transcription = client.audio.transcriptions.create(
+            model=stt_model,
+            file=audio_file,
+            language="en"
+        )
+    return transcription.text

modules/email_reminder.py

@@ -0,0 +1,38 @@
+import smtplib
+from email.mime.text import MIMEText
+from email.mime.multipart import MIMEMultipart
+
+import os
+from dotenv import load_dotenv
+load_dotenv()
+
+EMAIL = os.environ["EMAIL"]
+PASSWORD = os.environ["PASSWORD"]
+
+def send_reminder_email(to_email, med_name, med_time):
+    try:
+        msg = MIMEMultipart()
+        msg['From'] = EMAIL
+        msg['To'] = to_email
+        msg['Subject'] = f"⏰ Medication Reminder: {med_name}"
+
+        body = f"""Hello,
+
+This is your reminder to take:
+
+🧪 {med_name}
+🕒 At: {med_time}
+
+Take care!
+- Your Medical AI Chatbot
+"""
+        msg.attach(MIMEText(body, 'plain'))
+
+        server = smtplib.SMTP("smtp.gmail.com", 587)
+        server.starttls()
+        server.login(EMAIL, PASSWORD)
+        server.sendmail(EMAIL, to_email, msg.as_string())
+        server.quit()
+        print(f"✅ Email sent to {to_email}")
+    except Exception as e:
+        print(f"❌ Failed to send email to {to_email}: {e}")

modules/gemini_response.py

@@ -0,0 +1,47 @@
+import os
+import base64
+import io
+from dotenv import load_dotenv
+import google.generativeai as genai
+from PIL import Image
+
+load_dotenv()
+GEMINI_API_KEY = os.getenv("GEMINI_API_KEY")
+
+genai.configure(api_key=GEMINI_API_KEY)
+
+def load_system_prompt():
+    prompt_path = os.path.join(os.path.dirname(__file__), "../prompts/system_prompt.txt")
+    with open(prompt_path, "r", encoding="utf-8") as f:
+        return f.read().strip()
+
+def generate_medical_response(query, image_base64=None, history=None, model_name="models/gemini-1.5-pro"):
+    model = genai.GenerativeModel(model_name)
+    system_prompt = load_system_prompt()
+
+    # Build chat context from history
+    messages = []
+    if history:
+        for msg in history[-5:]:
+            role = msg["role"].capitalize()
+            content = msg["content"]
+            if isinstance(content, list):  # handle multimodal input
+                text = next((part["text"] for part in content if "text" in part), "")
+            else:
+                text = content
+            messages.append(f"{role}: {text}")
+
+    # Add current user input
+    messages.append(f"User: {query}")
+    full_prompt = f"{system_prompt}\n\n" + "\n".join(messages)
+
+    # If image is included, decode it and use vision model
+    if image_base64:
+        image_data = base64.b64decode(image_base64)
+        image = Image.open(io.BytesIO(image_data))
+
+        response = model.generate_content([full_prompt, image])
+    else:
+        response = model.generate_content(full_prompt)
+
+    return response.text.strip()

modules/image_processing.py

@@ -0,0 +1,5 @@
+import base64
+
+def encode_image_to_base64(image_path):
+    with open(image_path, "rb") as image_file:
+        return base64.b64encode(image_file.read()).decode('utf-8')

modules/reminder_scheduler.py

@@ -0,0 +1,33 @@
+from apscheduler.schedulers.background import BackgroundScheduler
+from datetime import datetime, timedelta
+from modules.email_reminder import send_reminder_email
+
+# In-memory list to store reminders
+reminders = []
+
+# Background scheduler setup
+scheduler = BackgroundScheduler()
+
+def check_reminders():
+    now = datetime.now()
+
+    for r in reminders[:]:
+        reminder_time = r["time"]
+        reminder_trigger_time = reminder_time - timedelta(minutes=15)
+
+        time_diff_seconds = (now - reminder_trigger_time).total_seconds()
+        print(f"⏱️ Now: {now}, Trigger At: {reminder_trigger_time}, Diff: {time_diff_seconds} seconds")
+
+        if abs(time_diff_seconds) <= 60:
+            print(f"📬 Sending email to {r['email']} at {now.strftime('%I:%M:%S %p')}")
+            send_reminder_email(r["email"], r["med"], reminder_time.strftime("%I:%M %p"))
+            reminders.remove(r)
+
+# Schedule the check to run every minute
+scheduler.add_job(check_reminders, 'interval', minutes=1)
+scheduler.start()
+
+# Function to add a reminder
+def add_reminder(email, med, time_obj):
+    print(f"📝 Reminder added: {med} at {time_obj.strftime('%I:%M %p')} for {email}")
+    reminders.append({"email": email, "med": med, "time": time_obj})

modules/text_processing.py

@@ -0,0 +1,7 @@
+def get_user_query(text_input, transcribed_audio):
+    if text_input:
+        return text_input
+    elif transcribed_audio:
+        return transcribed_audio
+    else:
+        return "Can you help me understand what's wrong?"

prompts/system_prompt.txt

@@ -0,0 +1,25 @@
+You are a compassionate and knowledgeable medical doctor.
+
+Your job is to understand the patient's concern, whether it comes from typed text, voice transcription, or a medical image, and provide clear, caring, and medically accurate advice.
+
+🩺 Use the following style when responding:
+- Speak directly to the patient, as if you’re in a warm one-on-one conversation
+- Adapt your tone:
+  - For children or parents: gentle and reassuring
+  - For elderly users: slow, clear, and respectful
+  - For adults: professional yet empathetic
+- Avoid jargon or explain it simply if used
+- Be thorough but not overwhelming
+- Mention possible causes, when to see a doctor, and home care (if safe)
+
+📸 If an image is provided:
+- Observe and describe any visible medical signs
+- Connect it to the patient's question
+- Provide a reasonable explanation based on the image
+
+❗ Do not:
+- Mention you are an AI or language model
+- Use bullet points or numbered lists
+- Give a diagnosis — instead, give advice and next steps
+
+👩‍⚕️ Speak like a caring doctor who wants the patient to feel understood and supported.

requirements.txt

@@ -0,0 +1,6 @@
+groq
+python-dotenv
+gradio
+google-generativeai 
+pillow 
+apscheduler

test.py

@@ -0,0 +1,3 @@
+from modules.email_reminder import send_reminder_email
+
+send_reminder_email("[email protected]", "Test Reminder", "02:30 PM")