Update app.py
Browse files
app.py
CHANGED
@@ -29,12 +29,12 @@ logging.basicConfig(level=logging.DEBUG, format="%(asctime)s - %(levelname)s - %
|
|
29 |
logger = logging.getLogger(__name__)
|
30 |
usage_metrics = {"total_assessments": 0, "assessments_by_language": {}}
|
31 |
|
32 |
-
# Environment variables
|
33 |
SF_USERNAME = os.getenv("SF_USERNAME", "[email protected]")
|
34 |
SF_PASSWORD = os.getenv("SF_PASSWORD", "voicebot1")
|
35 |
SF_SECURITY_TOKEN = os.getenv("SF_SECURITY_TOKEN", "jq4VVHUFti6TmzJDjjegv2h6b")
|
36 |
SF_INSTANCE_URL = os.getenv("SF_INSTANCE_URL", "https://swe42.sfdc-cehfhs.salesforce.com")
|
37 |
-
GEMINI_API_KEY = os.getenv("GEMINI_API_KEY", "
|
38 |
ENCRYPTION_KEY = os.getenv("ENCRYPTION_KEY", Fernet.generate_key().decode())
|
39 |
DEFAULT_EMAIL = os.getenv("SALESFORCE_USER_EMAIL", "[email protected]")
|
40 |
|
@@ -42,6 +42,7 @@ DEFAULT_EMAIL = os.getenv("SALESFORCE_USER_EMAIL", "[email protected]")
|
|
42 |
cipher = Fernet(ENCRYPTION_KEY)
|
43 |
|
44 |
# Initialize Salesforce
|
|
|
45 |
try:
|
46 |
sf = Salesforce(
|
47 |
username=SF_USERNAME,
|
@@ -52,17 +53,19 @@ try:
|
|
52 |
logger.info(f"Connected to Salesforce at {SF_INSTANCE_URL}")
|
53 |
except Exception as e:
|
54 |
logger.error(f"Salesforce connection failed: {str(e)}")
|
55 |
-
sf = None
|
56 |
|
57 |
# Initialize Google Gemini
|
58 |
-
|
59 |
-
|
60 |
-
|
61 |
-
|
62 |
-
|
63 |
-
|
64 |
-
|
65 |
-
|
|
|
|
|
|
|
66 |
|
67 |
# Load Whisper model
|
68 |
SUPPORTED_LANGUAGES = {"English": "english", "Hindi": "hindi", "Spanish": "spanish", "Mandarin": "mandarin"}
|
@@ -143,7 +146,7 @@ def cached_transcribe(audio_file, language):
|
|
143 |
def extract_health_features(audio, sr):
|
144 |
try:
|
145 |
audio = librosa.util.normalize(audio)
|
146 |
-
frame_duration =
|
147 |
frame_samples = int(sr * frame_duration / 1000)
|
148 |
frames = [audio[i:i + frame_samples] for i in range(0, len(audio), frame_samples)]
|
149 |
voiced_frames = [frame for frame in frames if len(frame) == frame_samples and vad.is_speech((frame * 32768).astype(np.int16).tobytes(), sr)]
|
@@ -151,19 +154,19 @@ def extract_health_features(audio, sr):
|
|
151 |
raise ValueError("No voiced segments detected")
|
152 |
voiced_audio = np.concatenate(voiced_frames)
|
153 |
|
154 |
-
frame_step = max(1, len(voiced_audio) // (sr //
|
155 |
pitches, magnitudes = librosa.piptrack(y=voiced_audio[::frame_step], sr=sr, fmin=75, fmax=300)
|
156 |
valid_pitches = [p for p in pitches[magnitudes > 0] if 75 <= p <= 300]
|
157 |
pitch = np.mean(valid_pitches) if valid_pitches else 0
|
158 |
jitter = np.std(valid_pitches) / pitch if pitch and valid_pitches else 0
|
159 |
jitter = min(jitter, 10)
|
160 |
-
amplitudes = librosa.feature.rms(y=voiced_audio, frame_length=
|
161 |
shimmer = np.std(amplitudes) / np.mean(amplitudes) if np.mean(amplitudes) else 0
|
162 |
shimmer = min(shimmer, 10)
|
163 |
energy = np.mean(amplitudes)
|
164 |
|
165 |
-
mfcc = np.mean(librosa.feature.mfcc(y=voiced_audio[::
|
166 |
-
spectral_centroid = np.mean(librosa.feature.spectral_centroid(y=voiced_audio[::
|
167 |
|
168 |
logger.debug(f"Extracted features: pitch={pitch:.2f}, jitter={jitter*100:.2f}%, shimmer={shimmer*100:.2f}%, energy={energy:.4f}, mfcc_mean={np.mean(mfcc):.2f}, spectral_centroid={spectral_centroid:.2f}")
|
169 |
return {
|
@@ -183,9 +186,9 @@ def transcribe_audio(audio, language="en"):
|
|
183 |
whisper_model.config.forced_decoder_ids = whisper_processor.get_decoder_prompt_ids(
|
184 |
language=SUPPORTED_LANGUAGES.get({"en": "English", "hi": "Hindi", "es": "Spanish", "zh": "Mandarin"}.get(language, "English"), "english"), task="transcribe"
|
185 |
)
|
186 |
-
inputs = whisper_processor(audio, sampling_rate=16000, return_tensors="pt")
|
187 |
with torch.no_grad():
|
188 |
-
generated_ids = whisper_model.generate(inputs["input_features"], max_new_tokens=
|
189 |
transcription = whisper_processor.batch_decode(generated_ids, skip_special_tokens=True)[0]
|
190 |
logger.info(f"Transcription (language: {language}): {transcription}")
|
191 |
return transcription
|
@@ -195,140 +198,104 @@ def transcribe_audio(audio, language="en"):
|
|
195 |
|
196 |
async def get_chatbot_response(message, language="en"):
|
197 |
if not chat or not message:
|
198 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
199 |
language_code = {"English": "en", "Hindi": "hi", "Spanish": "es", "Mandarin": "zh"}.get(language, "en")
|
200 |
-
full_context = "\n".join(context) + f"\nUser: {message}\nMindCare:
|
201 |
try:
|
202 |
-
response = await asyncio.get_event_loop().run_in_executor(None, lambda: chat.send_message(
|
|
|
|
|
|
|
|
|
|
|
|
|
203 |
with tempfile.NamedTemporaryFile(delete=False, suffix=".mp3") as temp_audio:
|
204 |
-
tts = gTTS(text=
|
205 |
tts.save(temp_audio.name)
|
206 |
audio_path = temp_audio.name
|
207 |
-
logger.info(f"Generated response: {
|
208 |
-
return
|
209 |
except Exception as e:
|
210 |
logger.error(f"Chatbot response failed: {str(e)}")
|
211 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
212 |
|
213 |
def analyze_symptoms(text, features):
|
214 |
feedback = []
|
215 |
-
suggestions = []
|
216 |
text = text.lower() if text else ""
|
217 |
|
218 |
-
# Generate health assessment feedback
|
219 |
if "cough" in text or "coughing" in text:
|
220 |
-
feedback.append("You mentioned a cough
|
221 |
-
suggestions.extend([
|
222 |
-
"• Drink warm fluids like herbal tea or water to soothe your throat.",
|
223 |
-
"• Rest to help your body recover from possible infection.",
|
224 |
-
"• Use a humidifier to ease throat irritation.",
|
225 |
-
"• Consider over-the-counter cough remedies, but consult a doctor first.",
|
226 |
-
"• Monitor symptoms; see a doctor if the cough lasts over a week."
|
227 |
-
])
|
228 |
elif "fever" in text or "temperature" in text:
|
229 |
-
feedback.append("You mentioned a fever
|
230 |
-
suggestions.extend([
|
231 |
-
"• Stay hydrated with water or electrolyte drinks.",
|
232 |
-
"• Rest to support your immune system.",
|
233 |
-
"• Monitor your temperature regularly.",
|
234 |
-
"• Use paracetamol to reduce fever, but follow dosage instructions.",
|
235 |
-
"• Seek medical advice if fever exceeds 100.4°F (38°C) for over 2 days."
|
236 |
-
])
|
237 |
elif "headache" in text:
|
238 |
-
feedback.append("You mentioned a headache
|
239 |
-
suggestions.extend([
|
240 |
-
"• Drink plenty of water to stay hydrated.",
|
241 |
-
"• Take short breaks to relax your mind.",
|
242 |
-
"• Try a mild pain reliever like ibuprofen, but consult a doctor.",
|
243 |
-
"• Practice deep breathing to reduce tension.",
|
244 |
-
"• Ensure you're getting enough sleep (7-8 hours)."
|
245 |
-
])
|
246 |
elif "stress" in text or "anxious" in text or "mental stress" in text:
|
247 |
-
feedback.append("You mentioned stress or anxiety,
|
248 |
-
suggestions.extend([
|
249 |
-
"• Try 5 minutes of deep breathing to calm your mind.",
|
250 |
-
"• Write in a journal to process your thoughts.",
|
251 |
-
"• Take a short walk in nature to relax.",
|
252 |
-
"• Practice mindfulness or meditation daily.",
|
253 |
-
"• Talk to a trusted friend or professional for support.",
|
254 |
-
"• Prioritize sleep and avoid excessive caffeine."
|
255 |
-
])
|
256 |
elif "respiratory" in text or "breathing" in text or "shortness of breath" in text:
|
257 |
-
feedback.append("You mentioned
|
258 |
-
suggestions.extend([
|
259 |
-
"• Avoid triggers like smoke or allergens.",
|
260 |
-
"• Practice slow, deep breathing exercises.",
|
261 |
-
"• Stay in a well-ventilated area.",
|
262 |
-
"• Monitor symptoms and seek medical help if severe.",
|
263 |
-
"• Rest to reduce strain on your respiratory system."
|
264 |
-
])
|
265 |
elif "cold" in text:
|
266 |
-
feedback.append("You mentioned a cold
|
267 |
-
suggestions.extend([
|
268 |
-
"• Drink warm fluids like soup or tea.",
|
269 |
-
"• Rest to help your body fight the virus.",
|
270 |
-
"• Use saline nasal spray to relieve congestion.",
|
271 |
-
"• Take over-the-counter cold remedies, but consult a doctor.",
|
272 |
-
"• Stay hydrated and avoid strenuous activity."
|
273 |
-
])
|
274 |
|
275 |
-
# Voice feature-based feedback and suggestions
|
276 |
if features["jitter"] > 6.5:
|
277 |
-
feedback.append(f"High jitter ({features['jitter']:.2f}%)
|
278 |
-
suggestions.append("• Rest your voice and avoid shouting.")
|
279 |
elif features["jitter"] > 4.0:
|
280 |
-
feedback.append(f"Moderate jitter ({features['jitter']:.2f}%)
|
281 |
-
suggestions.append("• Sip warm water to soothe your vocal cords.")
|
282 |
|
283 |
if features["shimmer"] > 7.5:
|
284 |
-
feedback.append(f"High shimmer ({features['shimmer']:.2f}%)
|
285 |
-
suggestions.append("• Try relaxation techniques like yoga or meditation.")
|
286 |
elif features["shimmer"] > 5.0:
|
287 |
-
feedback.append(f"Moderate shimmer ({features['shimmer']:.2f}%)
|
288 |
-
suggestions.append("• Stay hydrated to support vocal health.")
|
289 |
|
290 |
if features["energy"] < 0.003:
|
291 |
-
feedback.append(f"
|
292 |
-
suggestions.append("• Ensure 7-8 hours of sleep nightly.")
|
293 |
elif features["energy"] < 0.007:
|
294 |
-
feedback.append(f"Low vocal energy ({features['energy']:.4f}) suggests possible
|
295 |
-
suggestions.append("• Take short naps to boost energy.")
|
296 |
|
297 |
if features["pitch"] < 70 or features["pitch"] > 290:
|
298 |
-
feedback.append(f"Unusual pitch ({features['pitch']:.2f} Hz) may indicate vocal issues.")
|
299 |
-
suggestions.append("• Consult a doctor for a vocal health check.")
|
300 |
elif 70 <= features["pitch"] <= 90 or 270 <= features["pitch"] <= 290:
|
301 |
-
feedback.append(f"Pitch ({features['pitch']:.2f} Hz) is slightly outside typical range.")
|
302 |
-
suggestions.append("• Avoid straining your voice during conversations.")
|
303 |
|
304 |
if features["spectral_centroid"] > 2700:
|
305 |
-
feedback.append(f"High spectral centroid ({features['spectral_centroid']:.2f} Hz) suggests tense speech.")
|
306 |
-
suggestions.append("• Practice slow, calm speaking to reduce tension.")
|
307 |
elif features["spectral_centroid"] > 2200:
|
308 |
-
feedback.append(f"Elevated spectral centroid ({features['spectral_centroid']:.2f} Hz) may indicate mild tension.")
|
309 |
-
suggestions.append("• Relax your jaw and shoulders while speaking.")
|
310 |
|
311 |
if not feedback:
|
312 |
-
feedback.append("No significant health concerns detected from voice or text analysis.")
|
313 |
-
suggestions.extend([
|
314 |
-
"• Maintain a balanced diet with fruits and vegetables.",
|
315 |
-
"• Exercise regularly for overall health.",
|
316 |
-
"• Stay hydrated with 8 glasses of water daily.",
|
317 |
-
"• Get 7-8 hours of sleep each night.",
|
318 |
-
"• Practice stress-relief techniques like meditation.",
|
319 |
-
"• Schedule regular health check-ups."
|
320 |
-
])
|
321 |
|
322 |
-
|
323 |
-
|
324 |
-
if len(suggestions) < 6:
|
325 |
-
suggestions.extend([
|
326 |
-
"• Stay active with light exercise like walking.",
|
327 |
-
"• Practice gratitude to boost mental well-being."
|
328 |
-
][:6 - len(suggestions)])
|
329 |
-
|
330 |
-
logger.debug(f"Generated feedback: {feedback}, Suggestions: {suggestions}")
|
331 |
-
return "\n".join(feedback), "\n".join(suggestions)
|
332 |
|
333 |
def store_user_consent(email, language):
|
334 |
if not sf:
|
@@ -556,12 +523,16 @@ async def analyze_voice(audio_file=None, language="English", email=None):
|
|
556 |
usage_metrics["total_assessments"] += 1
|
557 |
usage_metrics["assessments_by_language"][language] = usage_metrics["assessments_by_language"].get(language, 0) + 1
|
558 |
|
|
|
|
|
|
|
|
|
559 |
try:
|
560 |
if not audio_file or not os.path.exists(audio_file):
|
561 |
raise ValueError("No valid audio file provided")
|
562 |
|
563 |
audio, sr = librosa.load(audio_file, sr=16000)
|
564 |
-
max_duration = 5
|
565 |
if len(audio) > max_duration * sr:
|
566 |
audio = audio[:max_duration * sr]
|
567 |
logger.info(f"Truncated audio to first {max_duration} seconds for faster processing")
|
@@ -570,7 +541,7 @@ async def analyze_voice(audio_file=None, language="English", email=None):
|
|
570 |
|
571 |
language_code = {"English": "en", "Hindi": "hi", "Spanish": "es", "Mandarin": "zh"}.get(language, "en")
|
572 |
user_id = store_user_consent(email, language)
|
573 |
-
if not user_id:
|
574 |
logger.warning("Proceeding with analysis despite consent storage failure")
|
575 |
feedback_message = "Warning: User consent could not be stored in Salesforce, but analysis will proceed.\n"
|
576 |
else:
|
@@ -578,7 +549,7 @@ async def analyze_voice(audio_file=None, language="English", email=None):
|
|
578 |
|
579 |
features = extract_health_features(audio, sr)
|
580 |
transcription = cached_transcribe(audio_file, language)
|
581 |
-
feedback
|
582 |
|
583 |
respiratory_score = features["jitter"]
|
584 |
mental_health_score = features["shimmer"]
|
@@ -594,6 +565,14 @@ async def analyze_voice(audio_file=None, language="English", email=None):
|
|
594 |
feedback += f"- Email: {email if email and email.strip() else DEFAULT_EMAIL}\n"
|
595 |
feedback += "\n**Disclaimer**: This is a preliminary analysis. Consult a healthcare provider for professional evaluation."
|
596 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
597 |
if user_id and sf:
|
598 |
store_in_salesforce(user_id, audio_file, feedback, respiratory_score, mental_health_score, features, transcription, language)
|
599 |
else:
|
@@ -602,16 +581,7 @@ async def analyze_voice(audio_file=None, language="English", email=None):
|
|
602 |
file_path, pdf_error = generate_pdf_report(feedback, transcription, features, language, email, suggestions)
|
603 |
if pdf_error:
|
604 |
feedback += f"\n\n**Error**: {pdf_error}"
|
605 |
-
return feedback, file_path, suggestions,
|
606 |
-
|
607 |
-
# Generate audio response based on suggestions
|
608 |
-
response_text = suggestions
|
609 |
-
response, audio_path = await get_chatbot_response(response_text, language)
|
610 |
-
if audio_path:
|
611 |
-
logger.info(f"Generated audio response at {audio_path}")
|
612 |
-
else:
|
613 |
-
logger.warning("Failed to generate audio response")
|
614 |
-
return feedback, file_path, response, None
|
615 |
|
616 |
try:
|
617 |
os.remove(audio_file)
|
@@ -619,7 +589,7 @@ async def analyze_voice(audio_file=None, language="English", email=None):
|
|
619 |
except Exception as e:
|
620 |
logger.error(f"Failed to delete audio file: {str(e)}")
|
621 |
|
622 |
-
return feedback, file_path,
|
623 |
except Exception as e:
|
624 |
logger.error(f"Audio processing failed: {str(e)}")
|
625 |
return f"Error: {str(e)}", None, "Error: Could not generate suggestions due to audio processing failure.", None
|
@@ -680,16 +650,7 @@ def launch():
|
|
680 |
"""
|
681 |
|
682 |
def check_microphone_access():
|
683 |
-
|
684 |
-
from navigator import mediaDevices
|
685 |
-
devices = mediaDevices.enumerateDevices()
|
686 |
-
for device in devices:
|
687 |
-
if device.kind == "audioinput":
|
688 |
-
return None
|
689 |
-
return "Microphone access is not available. Please upload an audio file or check browser permissions."
|
690 |
-
except Exception as e:
|
691 |
-
logger.error(f"Microphone access check failed: {str(e)}")
|
692 |
-
return "Microphone access is not available. Please upload an audio file or check browser permissions."
|
693 |
|
694 |
with gr.Blocks(title="MindCare Health Assistant", css=custom_css) as demo:
|
695 |
gr.Markdown("Record your voice or type a message for health assessments and suggestions.")
|
@@ -697,12 +658,11 @@ def launch():
|
|
697 |
with gr.Row():
|
698 |
with gr.Column():
|
699 |
gr.Markdown("### Voice Analysis")
|
700 |
-
mic_warning = gr.Markdown()
|
701 |
-
mic_warning.value = check_microphone_access() or ""
|
702 |
gr.Markdown("Upload voice (1+ sec) describing symptoms (e.g., 'I have a cough' or 'I feel stressed'). Note: Microphone recording may not be supported in all contexts; use file upload instead.")
|
703 |
email_input = gr.Textbox(label="Enter Your Email", placeholder="e.g., [email protected]", value="")
|
704 |
language_input = gr.Dropdown(choices=list(SUPPORTED_LANGUAGES.keys()), label="Select Language", value="English")
|
705 |
-
consent_input = gr.Checkbox(label="I consent to data storage and voice analysis", value=True, interactive=
|
706 |
audio_input = gr.Audio(type="filepath", label="Upload Voice (WAV, MP3, FLAC)", format="wav", interactive=True)
|
707 |
voice_output = gr.Textbox(label="Health Assessment Results", elem_id="health-results")
|
708 |
file_output = gr.File(label="Download Assessment Report (PDF)", file_types=[".pdf"])
|
@@ -741,4 +701,4 @@ def launch():
|
|
741 |
demo.launch(server_name="0.0.0.0", server_port=7860)
|
742 |
|
743 |
if __name__ == "__main__":
|
744 |
-
launch()
|
|
|
29 |
logger = logging.getLogger(__name__)
|
30 |
usage_metrics = {"total_assessments": 0, "assessments_by_language": {}}
|
31 |
|
32 |
+
# Environment variables with fallbacks
|
33 |
SF_USERNAME = os.getenv("SF_USERNAME", "[email protected]")
|
34 |
SF_PASSWORD = os.getenv("SF_PASSWORD", "voicebot1")
|
35 |
SF_SECURITY_TOKEN = os.getenv("SF_SECURITY_TOKEN", "jq4VVHUFti6TmzJDjjegv2h6b")
|
36 |
SF_INSTANCE_URL = os.getenv("SF_INSTANCE_URL", "https://swe42.sfdc-cehfhs.salesforce.com")
|
37 |
+
GEMINI_API_KEY = os.getenv("GEMINI_API_KEY", "your_gemini_api_key_here") # Placeholder, replace in Dockerfile
|
38 |
ENCRYPTION_KEY = os.getenv("ENCRYPTION_KEY", Fernet.generate_key().decode())
|
39 |
DEFAULT_EMAIL = os.getenv("SALESFORCE_USER_EMAIL", "[email protected]")
|
40 |
|
|
|
42 |
cipher = Fernet(ENCRYPTION_KEY)
|
43 |
|
44 |
# Initialize Salesforce
|
45 |
+
sf = None
|
46 |
try:
|
47 |
sf = Salesforce(
|
48 |
username=SF_USERNAME,
|
|
|
53 |
logger.info(f"Connected to Salesforce at {SF_INSTANCE_URL}")
|
54 |
except Exception as e:
|
55 |
logger.error(f"Salesforce connection failed: {str(e)}")
|
|
|
56 |
|
57 |
# Initialize Google Gemini
|
58 |
+
chat = None
|
59 |
+
if GEMINI_API_KEY and GEMINI_API_KEY != "your_gemini_api_key_here":
|
60 |
+
try:
|
61 |
+
genai.configure(api_key=GEMINI_API_KEY)
|
62 |
+
gemini_model = genai.GenerativeModel('gemini-1.5-flash')
|
63 |
+
chat = gemini_model.start_chat(history=[])
|
64 |
+
logger.info("Connected to Google Gemini")
|
65 |
+
except Exception as e:
|
66 |
+
logger.error(f"Google Gemini initialization failed: {str(e)}")
|
67 |
+
else:
|
68 |
+
logger.warning("GEMINI_API_KEY not set or invalid, using fallback suggestions")
|
69 |
|
70 |
# Load Whisper model
|
71 |
SUPPORTED_LANGUAGES = {"English": "english", "Hindi": "hindi", "Spanish": "spanish", "Mandarin": "mandarin"}
|
|
|
146 |
def extract_health_features(audio, sr):
|
147 |
try:
|
148 |
audio = librosa.util.normalize(audio)
|
149 |
+
frame_duration = 20
|
150 |
frame_samples = int(sr * frame_duration / 1000)
|
151 |
frames = [audio[i:i + frame_samples] for i in range(0, len(audio), frame_samples)]
|
152 |
voiced_frames = [frame for frame in frames if len(frame) == frame_samples and vad.is_speech((frame * 32768).astype(np.int16).tobytes(), sr)]
|
|
|
154 |
raise ValueError("No voiced segments detected")
|
155 |
voiced_audio = np.concatenate(voiced_frames)
|
156 |
|
157 |
+
frame_step = max(1, len(voiced_audio) // (sr // 6))
|
158 |
pitches, magnitudes = librosa.piptrack(y=voiced_audio[::frame_step], sr=sr, fmin=75, fmax=300)
|
159 |
valid_pitches = [p for p in pitches[magnitudes > 0] if 75 <= p <= 300]
|
160 |
pitch = np.mean(valid_pitches) if valid_pitches else 0
|
161 |
jitter = np.std(valid_pitches) / pitch if pitch and valid_pitches else 0
|
162 |
jitter = min(jitter, 10)
|
163 |
+
amplitudes = librosa.feature.rms(y=voiced_audio, frame_length=256, hop_length=128)[0]
|
164 |
shimmer = np.std(amplitudes) / np.mean(amplitudes) if np.mean(amplitudes) else 0
|
165 |
shimmer = min(shimmer, 10)
|
166 |
energy = np.mean(amplitudes)
|
167 |
|
168 |
+
mfcc = np.mean(librosa.feature.mfcc(y=voiced_audio[::2], sr=sr, n_mfcc=3), axis=1)
|
169 |
+
spectral_centroid = np.mean(librosa.feature.spectral_centroid(y=voiced_audio[::2], sr=sr, n_fft=256, hop_length=128))
|
170 |
|
171 |
logger.debug(f"Extracted features: pitch={pitch:.2f}, jitter={jitter*100:.2f}%, shimmer={shimmer*100:.2f}%, energy={energy:.4f}, mfcc_mean={np.mean(mfcc):.2f}, spectral_centroid={spectral_centroid:.2f}")
|
172 |
return {
|
|
|
186 |
whisper_model.config.forced_decoder_ids = whisper_processor.get_decoder_prompt_ids(
|
187 |
language=SUPPORTED_LANGUAGES.get({"en": "English", "hi": "Hindi", "es": "Spanish", "zh": "Mandarin"}.get(language, "English"), "english"), task="transcribe"
|
188 |
)
|
189 |
+
inputs = whisper_processor(audio[:16000 * 5], sampling_rate=16000, return_tensors="pt")
|
190 |
with torch.no_grad():
|
191 |
+
generated_ids = whisper_model.generate(inputs["input_features"], max_new_tokens=50)
|
192 |
transcription = whisper_processor.batch_decode(generated_ids, skip_special_tokens=True)[0]
|
193 |
logger.info(f"Transcription (language: {language}): {transcription}")
|
194 |
return transcription
|
|
|
198 |
|
199 |
async def get_chatbot_response(message, language="en"):
|
200 |
if not chat or not message:
|
201 |
+
fallback_suggestions = (
|
202 |
+
"- Consult a healthcare professional for personalized advice.\n"
|
203 |
+
"- Maintain a balanced diet.\n"
|
204 |
+
"- Stay hydrated.\n"
|
205 |
+
"- Get adequate sleep.\n"
|
206 |
+
"- Exercise regularly.\n"
|
207 |
+
"- Practice stress management techniques.\n"
|
208 |
+
"- Monitor symptoms daily.\n"
|
209 |
+
"- Seek support if needed."
|
210 |
+
)
|
211 |
+
with tempfile.NamedTemporaryFile(delete=False, suffix=".mp3") as temp_audio:
|
212 |
+
tts = gTTS(text=fallback_suggestions, lang={"English": "en", "Hindi": "hi", "Spanish": "es", "Mandarin": "zh"}.get(language, "en"), slow=False)
|
213 |
+
tts.save(temp_audio.name)
|
214 |
+
audio_path = temp_audio.name
|
215 |
+
logger.warning("Chatbot unavailable, using fallback suggestions")
|
216 |
+
return fallback_suggestions, audio_path
|
217 |
+
|
218 |
language_code = {"English": "en", "Hindi": "hi", "Spanish": "es", "Mandarin": "zh"}.get(language, "en")
|
219 |
+
full_context = "\n".join(context) + f"\nUser: {message}\nMindCare:"
|
220 |
try:
|
221 |
+
response = await asyncio.get_event_loop().run_in_executor(None, lambda: chat.send_message(
|
222 |
+
f"{full_context}\nBased on the health assessment, provide 8 specific health suggestions in bullet points:"
|
223 |
+
).text)
|
224 |
+
suggestions = "\n".join(["- " + line.strip() for line in response.split("\n") if line.strip()][:8])
|
225 |
+
if len(suggestions.split("\n")) < 8:
|
226 |
+
suggestions += "\n- Consult a healthcare professional for personalized advice.\n- Maintain a balanced diet.\n- Stay hydrated.\n- Get adequate sleep.\n- Exercise regularly.\n- Practice stress management techniques.\n- Monitor symptoms daily.\n- Seek support if needed."
|
227 |
+
|
228 |
with tempfile.NamedTemporaryFile(delete=False, suffix=".mp3") as temp_audio:
|
229 |
+
tts = gTTS(text=suggestions, lang=language_code, slow=False)
|
230 |
tts.save(temp_audio.name)
|
231 |
audio_path = temp_audio.name
|
232 |
+
logger.info(f"Generated response: {suggestions[:100]}... and audio at {audio_path}")
|
233 |
+
return suggestions, audio_path
|
234 |
except Exception as e:
|
235 |
logger.error(f"Chatbot response failed: {str(e)}")
|
236 |
+
fallback_suggestions = (
|
237 |
+
"- Consult a healthcare professional for personalized advice.\n"
|
238 |
+
"- Maintain a balanced diet.\n"
|
239 |
+
"- Stay hydrated.\n"
|
240 |
+
"- Get adequate sleep.\n"
|
241 |
+
"- Exercise regularly.\n"
|
242 |
+
"- Practice stress management techniques.\n"
|
243 |
+
"- Monitor symptoms daily.\n"
|
244 |
+
"- Seek support if needed."
|
245 |
+
)
|
246 |
+
with tempfile.NamedTemporaryFile(delete=False, suffix=".mp3") as temp_audio:
|
247 |
+
tts = gTTS(text=fallback_suggestions, lang=language_code, slow=False)
|
248 |
+
tts.save(temp_audio.name)
|
249 |
+
audio_path = temp_audio.name
|
250 |
+
return fallback_suggestions, audio_path
|
251 |
|
252 |
def analyze_symptoms(text, features):
|
253 |
feedback = []
|
|
|
254 |
text = text.lower() if text else ""
|
255 |
|
|
|
256 |
if "cough" in text or "coughing" in text:
|
257 |
+
feedback.append("You mentioned a cough. This could indicate a respiratory issue like a cold or bronchitis. Stay hydrated, rest, and consider consulting a doctor if it persists.")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
258 |
elif "fever" in text or "temperature" in text:
|
259 |
+
feedback.append("You mentioned a fever. This could be a sign of infection or illness. Monitor your temperature, stay hydrated, and seek medical advice if it exceeds 100.4°F (38°C).")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
260 |
elif "headache" in text:
|
261 |
+
feedback.append("You mentioned a headache. This could be due to stress, dehydration, or tension. Try resting, drinking water, and using a mild pain reliever like Paracetamol. Consult a doctor if severe.")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
262 |
elif "stress" in text or "anxious" in text or "mental stress" in text:
|
263 |
+
feedback.append("You mentioned stress or anxiety. Try deep breathing exercises or mindfulness. If persistent, consider speaking with a mental health professional for support.")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
264 |
elif "respiratory" in text or "breathing" in text or "shortness of breath" in text:
|
265 |
+
feedback.append("You mentioned respiratory issues or shortness of breath. This could indicate asthma or an infection. Seek medical attention if it worsens.")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
266 |
elif "cold" in text:
|
267 |
+
feedback.append("You mentioned a cold. This could be a viral infection. Rest, stay hydrated, and consider over-the-counter remedies like decongestants or honey for cough. Consult a doctor if symptoms worsen.")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
268 |
|
|
|
269 |
if features["jitter"] > 6.5:
|
270 |
+
feedback.append(f"High jitter ({features['jitter']:.2f}%) detected, suggesting potential vocal cord strain or respiratory issues. Consult a doctor.")
|
|
|
271 |
elif features["jitter"] > 4.0:
|
272 |
+
feedback.append(f"Moderate jitter ({features['jitter']:.2f}%) detected, indicating possible vocal instability. Monitor and rest your voice.")
|
|
|
273 |
|
274 |
if features["shimmer"] > 7.5:
|
275 |
+
feedback.append(f"High shimmer ({features['shimmer']:.2f}%) suggests possible emotional stress or vocal fatigue. Consider professional evaluation.")
|
|
|
276 |
elif features["shimmer"] > 5.0:
|
277 |
+
feedback.append(f"Moderate shimmer ({features['shimmer']:.2f}%) indicates mild vocal strain. Rest and hydrate recommended.")
|
|
|
278 |
|
279 |
if features["energy"] < 0.003:
|
280 |
+
feedback.append(f"Very low vocal energy ({features['energy']:.4f}) detected, possibly indicating fatigue or low mood. Rest and consult a doctor if needed.")
|
|
|
281 |
elif features["energy"] < 0.007:
|
282 |
+
feedback.append(f"Low vocal energy ({features['energy']:.4f}) suggests possible fatigue. Ensure adequate rest.")
|
|
|
283 |
|
284 |
if features["pitch"] < 70 or features["pitch"] > 290:
|
285 |
+
feedback.append(f"Unusual pitch ({features['pitch']:.2f} Hz) detected, which may indicate vocal cord issues. Consult a doctor.")
|
|
|
286 |
elif 70 <= features["pitch"] <= 90 or 270 <= features["pitch"] <= 290:
|
287 |
+
feedback.append(f"Pitch ({features['pitch']:.2f} Hz) is slightly outside typical range, possibly due to tension. Monitor your voice.")
|
|
|
288 |
|
289 |
if features["spectral_centroid"] > 2700:
|
290 |
+
feedback.append(f"High spectral centroid ({features['spectral_centroid']:.2f} Hz) suggests tense speech, potentially linked to stress or anxiety.")
|
|
|
291 |
elif features["spectral_centroid"] > 2200:
|
292 |
+
feedback.append(f"Elevated spectral centroid ({features['spectral_centroid']:.2f} Hz) may indicate mild tension in speech.")
|
|
|
293 |
|
294 |
if not feedback:
|
295 |
+
feedback.append("No significant health concerns detected from voice or text analysis. Maintain a healthy lifestyle and consult a doctor if symptoms arise.")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
296 |
|
297 |
+
logger.debug(f"Generated feedback: {feedback}")
|
298 |
+
return "\n".join(feedback)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
299 |
|
300 |
def store_user_consent(email, language):
|
301 |
if not sf:
|
|
|
523 |
usage_metrics["total_assessments"] += 1
|
524 |
usage_metrics["assessments_by_language"][language] = usage_metrics["assessments_by_language"].get(language, 0) + 1
|
525 |
|
526 |
+
async def timeout_handler():
|
527 |
+
await asyncio.sleep(20)
|
528 |
+
raise TimeoutError("Processing took too long (exceeded 20 seconds)")
|
529 |
+
|
530 |
try:
|
531 |
if not audio_file or not os.path.exists(audio_file):
|
532 |
raise ValueError("No valid audio file provided")
|
533 |
|
534 |
audio, sr = librosa.load(audio_file, sr=16000)
|
535 |
+
max_duration = 5
|
536 |
if len(audio) > max_duration * sr:
|
537 |
audio = audio[:max_duration * sr]
|
538 |
logger.info(f"Truncated audio to first {max_duration} seconds for faster processing")
|
|
|
541 |
|
542 |
language_code = {"English": "en", "Hindi": "hi", "Spanish": "es", "Mandarin": "zh"}.get(language, "en")
|
543 |
user_id = store_user_consent(email, language)
|
544 |
+
if not user_id and sf:
|
545 |
logger.warning("Proceeding with analysis despite consent storage failure")
|
546 |
feedback_message = "Warning: User consent could not be stored in Salesforce, but analysis will proceed.\n"
|
547 |
else:
|
|
|
549 |
|
550 |
features = extract_health_features(audio, sr)
|
551 |
transcription = cached_transcribe(audio_file, language)
|
552 |
+
feedback = analyze_symptoms(transcription, features)
|
553 |
|
554 |
respiratory_score = features["jitter"]
|
555 |
mental_health_score = features["shimmer"]
|
|
|
565 |
feedback += f"- Email: {email if email and email.strip() else DEFAULT_EMAIL}\n"
|
566 |
feedback += "\n**Disclaimer**: This is a preliminary analysis. Consult a healthcare provider for professional evaluation."
|
567 |
|
568 |
+
task = asyncio.create_task(get_chatbot_response(feedback, language))
|
569 |
+
timeout_task = asyncio.create_task(timeout_handler())
|
570 |
+
suggestions, suggestion_audio = await asyncio.gather(task, timeout_task, return_exceptions=True)
|
571 |
+
if isinstance(suggestions, Exception):
|
572 |
+
raise suggestions
|
573 |
+
if isinstance(suggestion_audio, TimeoutError):
|
574 |
+
raise TimeoutError("Health suggestions generation timed out after 20 seconds")
|
575 |
+
|
576 |
if user_id and sf:
|
577 |
store_in_salesforce(user_id, audio_file, feedback, respiratory_score, mental_health_score, features, transcription, language)
|
578 |
else:
|
|
|
581 |
file_path, pdf_error = generate_pdf_report(feedback, transcription, features, language, email, suggestions)
|
582 |
if pdf_error:
|
583 |
feedback += f"\n\n**Error**: {pdf_error}"
|
584 |
+
return feedback, file_path, suggestions, suggestion_audio
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
585 |
|
586 |
try:
|
587 |
os.remove(audio_file)
|
|
|
589 |
except Exception as e:
|
590 |
logger.error(f"Failed to delete audio file: {str(e)}")
|
591 |
|
592 |
+
return feedback, file_path, suggestions, suggestion_audio
|
593 |
except Exception as e:
|
594 |
logger.error(f"Audio processing failed: {str(e)}")
|
595 |
return f"Error: {str(e)}", None, "Error: Could not generate suggestions due to audio processing failure.", None
|
|
|
650 |
"""
|
651 |
|
652 |
def check_microphone_access():
|
653 |
+
return "Microphone access is not available. Please upload an audio file or check browser permissions."
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
654 |
|
655 |
with gr.Blocks(title="MindCare Health Assistant", css=custom_css) as demo:
|
656 |
gr.Markdown("Record your voice or type a message for health assessments and suggestions.")
|
|
|
658 |
with gr.Row():
|
659 |
with gr.Column():
|
660 |
gr.Markdown("### Voice Analysis")
|
661 |
+
mic_warning = gr.Markdown(value=check_microphone_access())
|
|
|
662 |
gr.Markdown("Upload voice (1+ sec) describing symptoms (e.g., 'I have a cough' or 'I feel stressed'). Note: Microphone recording may not be supported in all contexts; use file upload instead.")
|
663 |
email_input = gr.Textbox(label="Enter Your Email", placeholder="e.g., [email protected]", value="")
|
664 |
language_input = gr.Dropdown(choices=list(SUPPORTED_LANGUAGES.keys()), label="Select Language", value="English")
|
665 |
+
consent_input = gr.Checkbox(label="I consent to data storage and voice analysis", value=True, interactive=True)
|
666 |
audio_input = gr.Audio(type="filepath", label="Upload Voice (WAV, MP3, FLAC)", format="wav", interactive=True)
|
667 |
voice_output = gr.Textbox(label="Health Assessment Results", elem_id="health-results")
|
668 |
file_output = gr.File(label="Download Assessment Report (PDF)", file_types=[".pdf"])
|
|
|
701 |
demo.launch(server_name="0.0.0.0", server_port=7860)
|
702 |
|
703 |
if __name__ == "__main__":
|
704 |
+
launch()
|