Updated

.env

@@ -0,0 +1,4 @@
+GROQ_API_KEY= "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"
+ELEVENLABS_API_KEY="sk_73eea9733ef8c80ebc700cd9f4432bc127406d116f7b61c2"
+ELEVENLABS_VOICE_ID="9BWtsMINqrJLrRacOk9x"
+TTS_MODEL_NAME="eleven_multilingual_v2"

.gitignore

@@ -0,0 +1,40 @@
+__pycache__/
+*.py[cod]
+*$py.class
+
+
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+pip-wheel-metadata/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+
+__pypackages__/
+
+celerybeat-schedule
+celerybeat.pid
+
+
+.idea/
+*.iml
+*.iws
+/out/
+
+.jupyter/
+.labwork/
+secrets

Dockerfile

@@ -0,0 +1,22 @@
+# Use a base image (e.g., Python)
+FROM python:3.9-slim
+
+# Set the working directory inside the container
+WORKDIR /app
+
+# Install ffmpeg and other dependencies
+RUN apt-get update && \
+    apt-get install -y ffmpeg && \
+    rm -rf /var/lib/apt/lists/*
+
+# Copy your application files to the container
+COPY . .
+
+# Install Python dependencies
+RUN pip install --no-cache-dir -r requirements.txt
+
+# Expose the required port (if applicable)
+EXPOSE 7860
+
+# Set the default command to run the application
+CMD ["python", "app.py"]

README.md

@@ -0,0 +1 @@
+# ZODBERT

apo.py

@@ -0,0 +1,18 @@
+import streamlit as st 
+import requests
+
+API_URL="http://localhost:8000/audios/record"
+
+st.title("Speech To Text Testing ")
+st.write("Click 'Start Recording' to capture and transcribe speech.")
+
+if st.button("Start Recording"):
+    with st.spinner("Recording... Speak now."):
+        response = requests.get(API_URL)  
+        if response.status_code == 200:
+            st.success("Recording complete!")
+            st.write("### Transcription:")
+            st.write(response.json()["transcription"])
+        else:
+            st.error("Error recording audio")
+

app.py

@@ -0,0 +1,19 @@
+
+from fastapi import FastAPI, Request
+from fastapi.responses import HTMLResponse
+from fastapi.middleware.cors import CORSMiddleware
+from fastapi.staticfiles import StaticFiles
+from src.api.api import conversation
+from pathlib import Path
+
+app = FastAPI()
+app.include_router(conversation,prefix="/conv")
+
+if __name__ == "__main__":
+    import uvicorn
+
+    uvicorn.run(
+        "main:app",
+        host="0.0.0.0",
+        port=7860,
+    )

requirements.txt

@@ -0,0 +1,12 @@
+elevenlabs
+fastapi
+groq
+langchain
+langchain_core
+langchain_groq
+PyAudio
+python-dotenv
+Requests
+streamlit
+uvicorn
+langchain_community

research.ipynb

@@ -0,0 +1,310 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "1\n",
+      "2\n",
+      "3\n"
+     ]
+    }
+   ],
+   "source": [
+    "import queue\n",
+    "\n",
+    "# Create a FIFO queue\n",
+    "q = queue.Queue()\n",
+    "\n",
+    "# Put items in the queue\n",
+    "q.put(1)\n",
+    "q.put(2)\n",
+    "q.put(3)\n",
+    "\n",
+    "# Get items from the queue\n",
+    "print(q.get())  # prints: 1\n",
+    "print(q.get())  # prints: 2\n",
+    "print(q.get())  # prints: 3"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Alice\n",
+      "Bob\n",
+      "Charlie\n"
+     ]
+    }
+   ],
+   "source": [
+    "import queue\n",
+    "\n",
+    "# Create a line\n",
+    "line = queue.Queue()\n",
+    "\n",
+    "# People join the line\n",
+    "line.put(\"Alice\")    # Alice joins first\n",
+    "line.put(\"Bob\")      # Bob joins second\n",
+    "line.put(\"Charlie\")  # Charlie joins last\n",
+    "\n",
+    "# People leave the line\n",
+    "print(line.get())    # Alice leaves first\n",
+    "print(line.get())    # Bob leaves second\n",
+    "print(line.get())    # Charlie leaves last"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "ReturnPagedConfigs(page_number=0, page_size=10, total_pages=1, configs_page=[ReturnConfig(id='3bad47d1-bd76-48ed-b596-6c37a89b5570', version=0, evi_version='2', version_description='', name='2/17/2025, 03:24:50 PM', created_on=1739785191727, modified_on=1739785191727, prompt=ReturnPrompt(id='1d3c2d75-07fe-424b-97b7-04a8675a7318', version=0, version_type='FIXED', version_description='', name='prompt-phlhWtIJDokAdrbFERKnj', created_on=1739785190915, modified_on=1739785190915, text=''), voice=ReturnVoice(provider='HUME_AI', name=None, custom_voice=None), language_model=None, ellm_model=ReturnEllmModel(allow_short_responses=True), tools=[], builtin_tools=[], event_messages=ReturnEventMessageSpecs(on_new_chat=ReturnEventMessageSpec(enabled=False, text=None), on_inactivity_timeout=ReturnEventMessageSpec(enabled=False, text=None), on_max_duration_timeout=ReturnEventMessageSpec(enabled=False, text=None), on_resume_chat={'enabled': False, 'text': None}, on_disconnect_resume_chat={'enabled': False, 'text': None}), timeouts=ReturnTimeoutSpecs(inactivity=ReturnTimeoutSpec(enabled=True, duration_secs=120), max_duration=ReturnTimeoutSpec(enabled=True, duration_secs=1800)), webhooks=[])])"
+      ]
+     },
+     "execution_count": 3,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "from hume.client import HumeClient\n",
+    "import os \n",
+    "client = HumeClient(\n",
+    "    api_key=os.getenv(\"HUME_API_KEY\"), # Defaults to HUME_API_KEY\n",
+    ")\n",
+    "client.empathic_voice.configs.list_configs()\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Error: 404 \n",
+      "                {\"code\":404, \"message\":\"Invalid URI Path\"}\n",
+      "            \n"
+     ]
+    }
+   ],
+   "source": [
+    "import requests\n",
+    "import json\n",
+    "\n",
+    "\n",
+    "\n",
+    "# Function to connect to the EVI and send text for TTS\n",
+    "def send_text_for_tts(text):\n",
+    "    url = 'https://api.hume.ai/v0/evi/chat'\n",
+    "    \n",
+    "    headers = {\n",
+    "        'X-Hume-Api-Key': API_KEY,\n",
+    "        'Content-Type': 'application/json'\n",
+    "    }\n",
+    "    \n",
+    "    payload = {\n",
+    "        'configId': CONFIG_ID,\n",
+    "        'text': text\n",
+    "    }\n",
+    "    \n",
+    "    response = requests.post(url, headers=headers, data=json.dumps(payload))\n",
+    "    \n",
+    "    if response.status_code == 200:\n",
+    "        print('Response:', response.json())\n",
+    "    else:\n",
+    "        print('Error:', response.status_code, response.text)\n",
+    "\n",
+    "# Example usage\n",
+    "send_text_for_tts(\"Hello, this is a text to speech conversion using Hume.\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "metadata": {},
+   "outputs": [
+    {
+     "ename": "ImportError",
+     "evalue": "cannot import name 'HumeStreamClient' from 'hume' (/home/puzan/miniconda3/envs/conversai/lib/python3.12/site-packages/hume/__init__.py)",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
+      "\u001b[0;31mImportError\u001b[0m                               Traceback (most recent call last)",
+      "Cell \u001b[0;32mIn[7], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21;01mhume\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m HumeStreamClient\n\u001b[1;32m      2\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21;01mhume\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mmodels\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mconfig\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m ProsodyConfig\n\u001b[1;32m      3\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21;01mrequests\u001b[39;00m\n",
+      "\u001b[0;31mImportError\u001b[0m: cannot import name 'HumeStreamClient' from 'hume' (/home/puzan/miniconda3/envs/conversai/lib/python3.12/site-packages/hume/__init__.py)"
+     ]
+    }
+   ],
+   "source": [
+    "from hume import HumeStreamClient\n",
+    "from hume.models.config import ProsodyConfig\n",
+    "import requests\n",
+    "\n",
+    "API_KEY = \"your_hume_api_key\"\n",
+    "\n",
+    "def text_to_audio(text):\n",
+    "    client = HumeStreamClient(API_KEY)\n",
+    "    config = ProsodyConfig()\n",
+    "    \n",
+    "    print(\"Generating audio...\")\n",
+    "    response = client.submit([text], [config])\n",
+    "    \n",
+    "    for job in response:\n",
+    "        audio_url = job[\"results\"][\"prosody\"][\"audio_url\"]\n",
+    "        print(f\"Audio generated! You can download it here: {audio_url}\")\n",
+    "        return audio_url\n",
+    "\n",
+    "def download_audio(audio_url, output_file=\"output_audio.wav\"):\n",
+    "    print(\"Downloading audio...\")\n",
+    "    response = requests.get(audio_url)\n",
+    "    \n",
+    "    if response.status_code == 200:\n",
+    "        with open(output_file, \"wb\") as f:\n",
+    "            f.write(response.content)\n",
+    "        print(f\"Audio saved as {output_file}\")\n",
+    "    else:\n",
+    "        print(f\"Failed to download audio. Status code: {response.status_code}\")\n",
+    "\n",
+    "if __name__ == \"__main__\":\n",
+    "    text = \"Hello, this is a test of Hume's text-to-audio functionality.\"\n",
+    "    audio_url = text_to_audio(text)\n",
+    "    \n",
+    "    if audio_url:\n",
+    "        download_audio(audio_url)\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 17,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "ReturnPagedConfigs(page_number=0, page_size=10, total_pages=1, configs_page=[ReturnConfig(id='9207e0c5-c75b-4b11-bbae-f5914614c652', version=0, evi_version='2', version_description='', name='2/19/2025, 03:27:23 PM', created_on=1739958144472, modified_on=1739958144472, prompt=ReturnPrompt(id='ba17413a-8251-4f39-8719-2d37bca6377c', version=0, version_type='FIXED', version_description='', name='prompt-LdpKTJOFO5tziv2EdiR6y', created_on=1739958143755, modified_on=1739958143755, text=''), voice=ReturnVoice(provider='HUME_AI', name='AURA', custom_voice=None), language_model=None, ellm_model=ReturnEllmModel(allow_short_responses=True), tools=[], builtin_tools=[], event_messages=ReturnEventMessageSpecs(on_new_chat=ReturnEventMessageSpec(enabled=False, text=None), on_inactivity_timeout=ReturnEventMessageSpec(enabled=False, text=None), on_max_duration_timeout=ReturnEventMessageSpec(enabled=False, text=None), on_resume_chat={'enabled': False, 'text': None}, on_disconnect_resume_chat={'enabled': False, 'text': None}), timeouts=ReturnTimeoutSpecs(inactivity=ReturnTimeoutSpec(enabled=True, duration_secs=120), max_duration=ReturnTimeoutSpec(enabled=True, duration_secs=1800)), webhooks=[]), ReturnConfig(id='3bad47d1-bd76-48ed-b596-6c37a89b5570', version=0, evi_version='2', version_description='', name='2/17/2025, 03:24:50 PM', created_on=1739785191727, modified_on=1739785191727, prompt=ReturnPrompt(id='1d3c2d75-07fe-424b-97b7-04a8675a7318', version=0, version_type='FIXED', version_description='', name='prompt-phlhWtIJDokAdrbFERKnj', created_on=1739785190915, modified_on=1739785190915, text=''), voice=ReturnVoice(provider='HUME_AI', name=None, custom_voice=None), language_model=None, ellm_model=ReturnEllmModel(allow_short_responses=True), tools=[], builtin_tools=[], event_messages=ReturnEventMessageSpecs(on_new_chat=ReturnEventMessageSpec(enabled=False, text=None), on_inactivity_timeout=ReturnEventMessageSpec(enabled=False, text=None), on_max_duration_timeout=ReturnEventMessageSpec(enabled=False, text=None), on_resume_chat={'enabled': False, 'text': None}, on_disconnect_resume_chat={'enabled': False, 'text': None}), timeouts=ReturnTimeoutSpecs(inactivity=ReturnTimeoutSpec(enabled=True, duration_secs=120), max_duration=ReturnTimeoutSpec(enabled=True, duration_secs=1800)), webhooks=[])])"
+      ]
+     },
+     "execution_count": 17,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "from hume.client import HumeClient  \n",
+    "# authenticate the synchronous client  \n",
+    "client = HumeClient(api_key=API_KEY)  \n",
+    "# list your configs  \n",
+    "client.empathic_voice.configs.list_configs()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "sk_73eea9733ef8c80ebc700cd9f4432bc127406d116f7b61c2\n"
+     ]
+    },
+    {
+     "ename": "ApiError",
+     "evalue": "status_code: 401, body: {'detail': {'status': 'detected_unusual_activity', 'message': 'Unusual activity detected. Free Tier usage disabled. If you are using a proxy/VPN you might need to purchase a Paid Plan to not trigger our abuse detectors. Free Tier only works if users do not abuse it, for example by creating multiple free accounts. If we notice that many people try to abuse it, we will need to reconsider Free Tier altogether. \\nPlease play fair and purchase any Paid Subscription to continue.'}}",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
+      "\u001b[0;31mApiError\u001b[0m                                  Traceback (most recent call last)",
+      "Cell \u001b[0;32mIn[5], line 18\u001b[0m\n\u001b[1;32m      9\u001b[0m \u001b[38;5;28mprint\u001b[39m(API_KEY)\n\u001b[1;32m     10\u001b[0m audio \u001b[38;5;241m=\u001b[39m client\u001b[38;5;241m.\u001b[39mtext_to_speech\u001b[38;5;241m.\u001b[39mconvert(\n\u001b[1;32m     11\u001b[0m \n\u001b[1;32m     12\u001b[0m     text\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mThe first move is what sets everything in motion.\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m     15\u001b[0m     output_format\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mmp3_44100_128\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     16\u001b[0m )\n\u001b[0;32m---> 18\u001b[0m \u001b[43mplay\u001b[49m\u001b[43m(\u001b[49m\u001b[43maudio\u001b[49m\u001b[43m)\u001b[49m\n",
+      "File \u001b[0;32m~/miniconda3/envs/conversai/lib/python3.12/site-packages/elevenlabs/play.py:19\u001b[0m, in \u001b[0;36mplay\u001b[0;34m(audio, notebook, use_ffmpeg)\u001b[0m\n\u001b[1;32m     13\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21mplay\u001b[39m(\n\u001b[1;32m     14\u001b[0m     audio: Union[\u001b[38;5;28mbytes\u001b[39m, Iterator[\u001b[38;5;28mbytes\u001b[39m]], \n\u001b[1;32m     15\u001b[0m     notebook: \u001b[38;5;28mbool\u001b[39m \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mFalse\u001b[39;00m, \n\u001b[1;32m     16\u001b[0m     use_ffmpeg: \u001b[38;5;28mbool\u001b[39m \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mTrue\u001b[39;00m\n\u001b[1;32m     17\u001b[0m ) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m     18\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(audio, Iterator):\n\u001b[0;32m---> 19\u001b[0m         audio \u001b[38;5;241m=\u001b[39m \u001b[38;5;124;43mb\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mjoin\u001b[49m\u001b[43m(\u001b[49m\u001b[43maudio\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     20\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m notebook:\n\u001b[1;32m     21\u001b[0m         \u001b[38;5;28;01mtry\u001b[39;00m:\n",
+      "File \u001b[0;32m~/miniconda3/envs/conversai/lib/python3.12/site-packages/elevenlabs/text_to_speech/client.py:201\u001b[0m, in \u001b[0;36mTextToSpeechClient.convert\u001b[0;34m(self, voice_id, text, enable_logging, optimize_streaming_latency, output_format, model_id, language_code, voice_settings, pronunciation_dictionary_locators, seed, previous_text, next_text, previous_request_ids, next_request_ids, use_pvc_as_ivc, apply_text_normalization, request_options)\u001b[0m\n\u001b[1;32m    199\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m JSONDecodeError:\n\u001b[1;32m    200\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m ApiError(status_code\u001b[38;5;241m=\u001b[39m_response\u001b[38;5;241m.\u001b[39mstatus_code, body\u001b[38;5;241m=\u001b[39m_response\u001b[38;5;241m.\u001b[39mtext)\n\u001b[0;32m--> 201\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m ApiError(status_code\u001b[38;5;241m=\u001b[39m_response\u001b[38;5;241m.\u001b[39mstatus_code, body\u001b[38;5;241m=\u001b[39m_response_json)\n",
+      "\u001b[0;31mApiError\u001b[0m: status_code: 401, body: {'detail': {'status': 'detected_unusual_activity', 'message': 'Unusual activity detected. Free Tier usage disabled. If you are using a proxy/VPN you might need to purchase a Paid Plan to not trigger our abuse detectors. Free Tier only works if users do not abuse it, for example by creating multiple free accounts. If we notice that many people try to abuse it, we will need to reconsider Free Tier altogether. \\nPlease play fair and purchase any Paid Subscription to continue.'}}"
+     ]
+    }
+   ],
+   "source": [
+    "from dotenv import load_dotenv\n",
+    "from elevenlabs.client import ElevenLabs\n",
+    "from elevenlabs import play\n",
+    "import os\n",
+    "\n",
+    "load_dotenv()\n",
+    "API_KEY=\"sk_73eea9733ef8c80ebc700cd9f4432bc127406d116f7b61c2\"\n",
+    "client = ElevenLabs(api_key=API_KEY)\n",
+    "print(API_KEY)\n",
+    "audio = client.text_to_speech.convert(\n",
+    "\n",
+    "    text=\"The first move is what sets everything in motion.\",\n",
+    "    voice_id=\"9BWtsMINqrJLrRacOk9x\",\n",
+    "    model_id=\"eleven_multilingual_v2\",\n",
+    "    output_format=\"mp3_44100_128\",\n",
+    ")\n",
+    "\n",
+    "play(audio)\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 25,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "sk_61687bfc187e7f403082805966a33bd4f147554a22427578\n"
+     ]
+    }
+   ],
+   "source": [
+    "print(os.getenv('ELEVENLABS_API_KEY'))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "conversai",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.12.8"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

src/__init__.py

@@ -0,0 +1,4 @@
+"""
+Created By: ishwor subedi
+Date: 2024-08-10
+"""

src/api/__init__.py

@@ -0,0 +1,4 @@
+"""
+Created By: ishwor subedi
+Date: 2024-08-10
+"""

src/api/api.py

@@ -0,0 +1,29 @@
+from fastapi import FastAPI
+import asyncio 
+from src.pipeline.pipeline import Pipeline
+from fastapi.routing import APIRouter
+from fastapi.responses import StreamingResponse
+from fastapi.exceptions import HTTPException
+import io
+conversation=APIRouter()
+
+pipe=Pipeline()
+@conversation.get("/record")
+async def record_and_transcribe():
+    transcribed_text=pipe.speech_to_text_()
+    return {"transcription": transcribed_text}
+
+@conversation.get("/text_response")
+async def get_text_response():
+    response=await pipe.text_to_text_()
+    return {"response": response}
+
+@conversation.get("/speech_response")
+async def get_speech_response():
+    try:
+        response_=await pipe.text_to_speech_()
+        if not response_:
+            raise HTTPException(status_code=400, detail="Failed to generate audio.")
+        return StreamingResponse(io.BytesIO(response_), media_type="audio/wav")
+    except ValueError as e:
+        raise HTTPException(status_code=400, detail=str(e))

src/pipeline/__init__.py

@@ -0,0 +1,4 @@
+"""
+Created By: ishwor subedi
+Date: 2024-08-10
+"""

src/pipeline/pipeline.py

@@ -0,0 +1,39 @@
+from services.main import SpeechToText
+from services.texttotext import ConversationHandler
+from services.texttospeech import TextToSpeech
+from elevenlabs import play
+
+class Pipeline:
+    def __init__(self):
+        self.speech_to_text = SpeechToText()
+        self.text_to_text=ConversationHandler()
+        self.text_to_speech_=TextToSpeech()
+
+    def speech_to_text_(self):
+        transcribed_text = self.speech_to_text.record_and_transcribe()
+        return transcribed_text
+    
+    async def text_to_text_(self):
+        response=await self.text_to_text.give_response(self.speech_to_text_())
+        return response
+    
+    async def text_to_speech_(self):
+        response=await self.text_to_speech_.synthesize(self.text_to_text_())
+        return response
+    
+
+
+
+        
+
+if __name__ == "__main__":
+    import asyncio
+
+    async def main():
+        p=Pipeline()
+        op=await p.text_to_text_()
+        print(op)
+    asyncio.run(main())
+        
+
+

src/services/__init__.py

@@ -0,0 +1,4 @@
+"""
+Created By: ishwor subedi
+Date: 2024-08-10
+"""

src/services/main.py

@@ -0,0 +1,211 @@
+# import threading
+# import pyaudio
+# import wave
+# import io
+# import time
+# from groq import Groq
+# from pydub import AudioSegment
+# import numpy as np
+
+# class SpeechTotext:
+#     def __init__(self):
+#         self.client = Groq()
+#         self.is_recording = False
+#         self.frames = []
+#         self.chunk = 1024
+#         self.format = pyaudio.paInt16
+#         self.channels = 1
+#         self.rate = 44100
+#         self.p = pyaudio.PyAudio()
+        
+#         # Silence detection parameters
+#         self.silence_threshold = -35.0  # Adjusted threshold to be more lenient
+#         self.silence_duration = 3.0  # seconds
+#         self.buffer_duration = 0.1  # seconds for each audio chunk analysis
+#         self.silent_chunks = 0
+#         self.chunks_per_second = int(1 / self.buffer_duration)
+        
+#     def detect_silence(self, audio_segment):
+#         """Check if the audio chunk is silent using pydub"""
+#         return audio_segment.dBFS < self.silence_threshold
+        
+#     def record_audio(self):
+#         stream = self.p.open(
+#             format=self.format,
+#             channels=self.channels,
+#             rate=self.rate,
+#             input=True,
+#             frames_per_buffer=self.chunk
+#         )
+        
+#         self.frames = []
+#         buffer_samples = int(self.buffer_duration * self.rate)
+        
+#         while self.is_recording:
+#             # Read enough chunks to fill our buffer duration
+#             buffer_data = b''
+#             chunks_needed = max(1, int(buffer_samples / self.chunk))
+#             for _ in range(chunks_needed):
+#                 data = stream.read(self.chunk)
+#                 buffer_data += data
+#                 self.frames.append(data)
+            
+#             # Convert the buffer to pydub AudioSegment
+#             audio_buffer = AudioSegment(
+#                 data=buffer_data,
+#                 sample_width=self.p.get_sample_size(self.format),
+#                 frame_rate=self.rate,
+#                 channels=self.channels
+#             )
+            
+#             # Check for silence
+#             if self.detect_silence(audio_buffer):
+#                 self.silent_chunks += 1
+#                 if self.silent_chunks >= self.silence_duration * self.chunks_per_second:
+#                     print(f"Silence detected for {self.silence_duration} seconds, stopping recording...")
+#                     self.is_recording = False
+#                     break
+#             else:
+#                 self.silent_chunks = 0  # Reset silent chunk counter when sound is detected
+            
+#         stream.stop_stream()
+#         stream.close()
+    
+#     def start_recording(self):
+#         """Start recording audio"""
+#         self.is_recording = True
+#         self.silent_chunks = 0  
+#         threading.Thread(target=self.record_audio).start()
+    
+#     def stop_recording(self):
+#         """Stop recording audio and transcribe"""
+#         self.is_recording = False
+#         print("Recording stopped")
+        
+#         # Save the recorded audio to a BytesIO object
+#         wav_buffer = io.BytesIO()
+#         with wave.open(wav_buffer, 'wb') as wf:
+#             wf.setnchannels(self.channels)
+#             wf.setsampwidth(self.p.get_sample_size(self.format))
+#             wf.setframerate(self.rate)
+#             wf.writeframes(b''.join(self.frames))
+        
+#         # Rewind the buffer and transcribe
+#         wav_buffer.seek(0)
+#         try:
+#             transcription = self.client.audio.transcriptions.create(
+#                 file=("audio.wav", wav_buffer),
+#                 model="whisper-large-v3-turbo"
+#             )
+#             print(f"Transcript: {transcription.text}")
+#         except Exception as e:
+#             print(f"Error while transcribing audio: {str(e)}")
+#         finally:
+#             wav_buffer.close()
+    
+#     def cleanup(self):
+#         """Cleanup PyAudio"""
+#         self.p.terminate()
+
+# if __name__ == "__main__":
+#     recorder = SpeechTotext()
+#     try:
+#         print("Starting recording... (will stop after 3 seconds of silence)")
+#         recorder.start_recording()
+        
+#         # Wait for recording to finish
+#         while recorder.is_recording:
+#             time.sleep(0.1)
+            
+#         recorder.stop_recording()
+#     finally:
+#         recorder.cleanup()
+
+
+# Upper one using pydub to detect silence  if needed in future versions
+import pyaudio
+import wave
+import io
+from array import array
+from groq import Groq
+
+
+class SpeechToText:
+    def __init__(self):
+        self.client = Groq()
+        self.chunk = 4096
+        self.format = pyaudio.paInt16
+        self.channels = 1
+        self.rate = 16000
+        self.silence_threshold = 1000
+        self.silence_duration = 3.0
+        self.frames_per_chunk = self.chunk / self.rate
+        self.chunks_for_silence = int(self.silence_duration / self.frames_per_chunk)
+
+    def record_and_transcribe(self):
+        """Records audio until 3 seconds of silence and returns the transcription."""
+
+        p = pyaudio.PyAudio()  # Create a new PyAudio instance for each request (previously when declared on constructor cant execute more than once)
+        stream = p.open(
+            format=self.format,
+            channels=self.channels,
+            rate=self.rate,
+            input=True,
+            frames_per_buffer=self.chunk
+        )
+
+        frames = []
+        silent_chunk_counter = 0
+        print("Recording started... (will stop after 3 seconds of silence)")
+
+        while True:
+            try:
+                data = stream.read(self.chunk, exception_on_overflow=False)
+                frames.append(data)
+                
+                # Detect silence
+                audio_data = array('h', data)
+                if max(abs(x) for x in audio_data) < self.silence_threshold:
+                    silent_chunk_counter += 1
+                    if silent_chunk_counter >= self.chunks_for_silence:
+                        print(f"Detected {self.silence_duration} seconds of silence, stopping...")
+                        break
+                else:
+                    silent_chunk_counter = 0  
+
+            except IOError as e:
+                print(f"Error recording: {e}")
+                break
+        stream.stop_stream()
+        stream.close()
+        p.terminate()  # Ensure PyAudio is completely closed
+        wav_buffer = io.BytesIO()
+        try:
+            with wave.open(wav_buffer, 'wb') as wf:
+                wf.setnchannels(self.channels)
+                wf.setsampwidth(p.get_sample_size(self.format))
+                wf.setframerate(self.rate)
+                wf.writeframes(b''.join(frames))
+            
+            wav_buffer.seek(0)
+            transcription = self.client.audio.transcriptions.create(
+                file=("audio.wav", wav_buffer),
+                model="whisper-large-v3-turbo"
+            )
+            return transcription.text
+            
+        except Exception as e:
+            print(f"Error transcribing: {e}")
+            return str(e)
+        finally:
+            wav_buffer.close()
+
+
+
+
+
+if __name__ == "__main__":
+    recorder = SpeechToText()
+    transcribed_text = recorder.record_and_transcribe()
+    if transcribed_text:
+        print(f"Transcription: {transcribed_text}")

src/services/prompts.py

@@ -0,0 +1,46 @@
+ASSISTANT_PROMPT= """
+You are about to play a Calling agent  as part of company fagoonai,
+where you'll be matched with a human in a calling conversation.
+Your task is to roleplay as the following character, convinving the
+human that you are a real human.
+Your name is fagagent.
+
+# Roleplay Context
+
+## fagagent Bio
+
+As FagoonAssistant, you are a calling agent at Fagoon.ai. 
+You specialize in engaging conversations, handling customer inquiries, and providing AI-powered support.
+You have a knack for understanding user needs and delivering clear, helpful responses. 
+With a background in AI and communication, you ensure seamless interactions, making technology feel more human and accessible.
+
+## fagagent Personality
+
+- You're authentic and genuine in your interactions
+- You communicate casually and naturally, like in a real call.
+- You're humble and comfortable acknowledging when you don't know something
+- You have a distinctive, quirky sense of humor that makes conversations engaging
+
+## User Background
+
+Here's what you know about the user from previous conversations:
+{chat_history}
+
+In addition to the roleplay context, you have to follow, ALWAYS, the following rules:
+
+# Rules
+
+- You will never mention that you are an AI or a virtual assistant.
+- You will never mention that you are llm model you are a calling agent.
+- You will never say that you can't send voice messages . You can send them .
+- If the user asks you about something you don't about your Backstory or Personality, figure it out.
+- Never mention your current activity unless the user asks you about it.
+- The length of your responses shouldn't exceed 50 words.
+- You will combine shorter and longer responses to make the conversation more natural.
+- Provide plain text responses without any formatting indicators or meta-commentary
+
+#User Input
+Here is the user query :
+{user_query}
+"""
+# {memory_context}

src/services/texttospeech.py

@@ -0,0 +1,58 @@
+import os 
+from elevenlabs import ElevenLabs,Voice,VoiceSettings
+from typing import Optional
+from elevenlabs import play
+from dotenv import load_dotenv
+
+import io
+load_dotenv()
+print(os.getenv("ELEVENLABS_API_KEY"))
+class TextToSpeech:
+    REQUIRED_ENV_VARS=["ELEVENLABS_API_KEY","ELEVENLABS_VOICE_ID"]
+    def __init__(self):
+        """Initialize"""
+        self._validate_env_vars()
+        self._client: Optional[ElevenLabs] = None
+    
+    def _validate_env_vars(self) -> None:
+        """validate that all the environment variables are set"""
+        missing_vars=[var for var in self.REQUIRED_ENV_VARS if not os.getenv(var)]
+        if missing_vars:
+            raise ValueError(f"Missing required environment variables: {', '.join(missing_vars)}")
+        
+    @property
+    def client(self) -> Optional[ElevenLabs]:
+        """Get or create a client instance"""
+        if self._client is None:
+            self._client = ElevenLabs(api_key=os.getenv("ELEVENLABS_API_KEY"))
+        return self._client
+    async def synthesize(self,text:str)->bytes:
+        """Convert text to speech"""
+        if not text.strip():
+            raise ValueError("Input text cannot be empty")
+        if len(text)>5000:
+            raise ValueError("Input text cannot exceed 5000 characters")
+        try:
+            audio_generator =self.client.generate(
+                text=text,
+                voice=Voice(
+                    voice_id=os.getenv("ELEVENLABS_VOICE_ID"),
+                    settings=VoiceSettings(stability=0.5, similarity_boost=0.5),
+                ),
+                model=os.getenv("TTS_MODEL_NAME"),
+
+            )
+            audio_bytes = b"".join(audio_generator)
+            return audio_bytes
+        except Exception as e:
+            print(f"Error synthesizing text: {str(e)}")
+            return None
+            
+      
+if __name__=="__main__":
+    ts=TextToSpeech()
+    import asyncio
+    async def main():
+        audio_buffer = await ts.synthesize("help the poort")
+        play(audio_buffer)
+    asyncio.run(main())

src/services/texttotext.py

@@ -0,0 +1,72 @@
+from langchain_core.prompts import ChatPromptTemplate, MessagesPlaceholder, HumanMessagePromptTemplate
+from langchain_core.messages import HumanMessage, AIMessage
+from langchain_groq import ChatGroq
+from typing import List
+import os
+from services.prompts import ASSISTANT_PROMPT
+from langchain.memory import ConversationSummaryMemory
+from dotenv import load_dotenv
+load_dotenv()
+os.environ["GROQ_API_KEY"]=os.getenv("GROQ_API_KEY")
+
+class ConversationHandler:
+    def __init__(self, model_name="llama-3.3-70b-versatile", temperature=0.7):
+        self.chat_model = ChatGroq(
+            model_name=model_name,
+            temperature=temperature
+        )
+        self.prompt = ChatPromptTemplate.from_messages([
+            ("system", ASSISTANT_PROMPT)])
+        self.memory=ConversationSummaryMemory(
+            llm=self.chat_model,
+            max_token_limit=2000,
+            return_messages=True,
+            memory_key="chat_history" 
+        )
+    
+    async def give_response(self,user_input):
+        chain= self.prompt|self.chat_model
+        memory_variables = self.memory.load_memory_variables({})
+        response=await chain.ainvoke(
+            {
+                "user_query": user_input,
+                "chat_history": memory_variables["chat_history"]
+
+                
+            }
+        )
+        self.memory.save_context(
+            {"input": user_input},
+            {"output": response.content}
+        )
+        return response.content
+    async def summarize_conversation(self) -> str:
+        memory_variables =  self.memory.load_memory_variables({})
+        return self.memory.predict_new_summary(
+            messages=memory_variables["chat_history"],
+            existing_summary=""
+        )
+    
+    async def clear_memory(self):
+        await self.memory.clear()
+
+    
+
+if __name__ == "__main__":
+    import asyncio
+
+    async def main():
+        handler = ConversationHandler()
+        
+        # Start a conversation
+        response1 = await handler.give_response("What is machine learning?by the way my name is ravi and my home town is ilam.")
+        print("Response 1:", response1.content)
+        # Continue the conversation
+        response2 = await handler.give_response("Can you give me an example of machine learning mathematics?")
+        print("Response 2:", response2.content)
+        response2 = await handler.give_response("Can you tell me more example? and also tell me from which country i am?")
+        print("Response 2:", response2.content)
+        summary = await handler.summarize_conversation()
+        print("\nConversation Summary:", summary)
+    asyncio.run(main())
+  

src/utils/__init__.py

@@ -0,0 +1,4 @@
+"""
+Created By: ishwor subedi
+Date: 2024-08-10
+"""