add fastapi to host websocket

Files changed (6) hide show

frontend/index.html +47 -1
main.py +70 -0
pyproject.toml +1 -0
transcribe/pipelines/pipe_vad.py +3 -3
transcribe/whisper_llm_serve.py +12 -19
uv.lock +36 -0

frontend/index.html CHANGED Viewed

@@ -10,6 +10,52 @@
   </head>
   <body>
     <div id="app"></div>
   </body>
 </html>

   </head>
   <body>
     <div id="app"></div>
   </body>
+  <script>
+    async function startRecording() {
+      const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
+      const audioContext = new AudioContext({ sampleRate: 16000 });
+      const source = audioContext.createMediaStreamSource(stream);
+      const processor = audioContext.createScriptProcessor(4096, 1, 1);
+      const wsUrl = "ws://localhost:9090/ws?from=zh&to=en";
+      ws = new WebSocket(wsUrl);
+      ws.binaryType = "arraybuffer";
+      ws.onopen = () => {
+        console.log("WebSocket opened");
+        source.connect(processor);
+        processor.connect(audioContext.destination);
+        processor.onaudioprocess = (e) => {
+          const input = e.inputBuffer.getChannelData(0);
+          const buffer = new Int16Array(input.length);
+          for (let i = 0; i < input.length; i++) {
+            buffer[i] = Math.max(-1, Math.min(1, input[i])) * 0x7FFF;
+          }
+          ws.send(buffer);
+        };
+      };
+      ws.onmessage = (event) => {
+        try {
+          const msg = JSON.parse(event.data);
+          if (msg.result) {
+            addTranslation(msg.result);
+          }
+        } catch (e) {
+          console.error("Parse error:", e);
+        }
+      };
+      ws.onerror = (e) => console.error("WebSocket error:", e);
+      ws.onclose = () => {
+        console.log("WebSocket closed");
+        processor.disconnect();
+        source.disconnect();
+      };
+    }
+  </script>
+</body>
 </html>

main.py CHANGED Viewed

	@@ -0,0 +1,70 @@

+from fastapi import FastAPI, WebSocket
+from urllib.parse import urlparse, parse_qsl
+from transcribe.whisper_llm_serve import PyWhiperCppServe
+from uuid import uuid1
+from logging import getLogger
+import numpy as np
+from transcribe.translatepipes import TranslatePipes
+from contextlib import asynccontextmanager
+from multiprocessing import Process, freeze_support
+from fastapi.staticfiles import StaticFiles
+logger = getLogger(__name__)
+async def get_audio_from_websocket(websocket)->np.array:
+    """
+    Receives audio buffer from websocket and creates a numpy array out of it.
+    Args:
+        websocket: The websocket to receive audio from.
+    Returns:
+        A numpy array containing the audio.
+    """
+    frame_data = await websocket.receive_bytes()
+    if frame_data == b"END_OF_AUDIO":
+        return False
+    return np.frombuffer(frame_data, dtype=np.int16).astype(np.float32) / 32768.0
+@asynccontextmanager
+async def lifespan(app:FastAPI):
+    global pipe
+    pipe = TranslatePipes()
+    pipe.wait_ready()
+    logger.info("Pipeline is ready.")
+    yield
+app = FastAPI(lifespan=lifespan)
+app.mount("/translate", StaticFiles(directory="frontend"),)
+pipe = None
+@app.websocket("/ws")
+async def translate(websocket: WebSocket):
+    query_parameters_dict = websocket.query_params
+    from_lang, to_lang = query_parameters_dict.get('from'), query_parameters_dict.get('to')
+    client = PyWhiperCppServe(
+        websocket,
+        pipe,
+        language="en",
+        client_uid=f"{uuid1()}",
+    )
+    if from_lang and to_lang:
+        client.set_lang(from_lang, to_lang)
+        logger.info(f"Source lange: {from_lang}  -> Dst lange: {to_lang}")
+        await websocket.accept()
+    while True:
+        frame_data = await get_audio_from_websocket(websocket)
+        client.add_frames(frame_data)
+if __name__ == '__main__':
+    freeze_support()
+    import uvicorn
+    uvicorn.run(app, host="0.0.0.0", port=9090)

pyproject.toml CHANGED Viewed

@@ -18,6 +18,7 @@ dependencies = [
     "soundfile>=0.13.1",
     "torch>=2.6.0",
     "tqdm>=4.67.1",
     "websocket-client>=1.8.0",
     "websockets>=15.0.1",
 ]

     "soundfile>=0.13.1",
     "torch>=2.6.0",
     "tqdm>=4.67.1",
+    "uvicorn>=0.34.0",
     "websocket-client>=1.8.0",
     "websockets>=15.0.1",
 ]

transcribe/pipelines/pipe_vad.py CHANGED Viewed

@@ -4,7 +4,7 @@ from ..helpers.vadprocessor import SileroVADProcessor, FixedVADIterator
 import numpy as np
 from silero_vad import get_speech_timestamps,collect_chunks
 import torch
-import noisereduce as nr
 class VadPipe(BasePipe):
@@ -34,8 +34,8 @@ class VadPipe(BasePipe):
         return np.array([], dtype=np.float32)
-    def reduce_noise(self, data):
-        return nr.reduce_noise(y=data, sr=self.sample_rate)
     def process(self, in_data: MetaItem) -> MetaItem:

 import numpy as np
 from silero_vad import get_speech_timestamps,collect_chunks
 import torch
+# import noisereduce as nr
 class VadPipe(BasePipe):
         return np.array([], dtype=np.float32)
+    # def reduce_noise(self, data):
+    #     return nr.reduce_noise(y=data, sr=self.sample_rate)
     def process(self, in_data: MetaItem) -> MetaItem:

transcribe/whisper_llm_serve.py CHANGED Viewed

@@ -1,10 +1,8 @@
-import soundfile
-import multiprocessing as mp
 import numpy as np
 from logging import getLogger
 from .utils import save_to_wave
 import time
 import json
@@ -19,16 +17,11 @@ from .strategy import TripleTextBuffer, SegmentManager, segments_split, sequence
 logger = getLogger("TranslatorApp")
-translate_pipes = TranslatePipes()
-translate_pipes.wait_ready()
-logger.info("Pipeline is ready.")
 class PyWhiperCppServe(ServeClientBase):
-    def __init__(self, websocket, language=None, dst_lang=None, client_uid=None,):
         super().__init__(client_uid, websocket)
         self.language = language
         self.dst_lang = dst_lang # 目标翻译语言
@@ -36,7 +29,7 @@ class PyWhiperCppServe(ServeClientBase):
         self._text_buffer = TripleTextBuffer()
         # 存储转录数据
         self._segment_manager = SegmentManager()
         self.lock = threading.Lock()
         self.frames_np = None
         self._frame_queue = queue.Queue()
@@ -71,7 +64,7 @@ class PyWhiperCppServe(ServeClientBase):
     def vad_merge(self):
         with self.lock:
             frame = self.frames_np.copy()
-            item = translate_pipes.voice_detect(frame.tobytes())
             if item.audio != b'':
                 frame_np = np.frombuffer(item.audio, dtype=np.float32)
                 self.frames_np = frame_np.copy()
@@ -105,7 +98,7 @@ class PyWhiperCppServe(ServeClientBase):
         log_block("Audio buffer length", f"{audio_buffer.shape[0]/self.sample_rate:.2f}", "s")
         start_time = time.perf_counter()
-        item = translate_pipes.transcrible(audio_buffer.tobytes(), self.language)
         segments = item.segments
         log_block("Whisper transcrible time", f"{(time.perf_counter() - start_time):.3f}", "s")
@@ -117,7 +110,7 @@ class PyWhiperCppServe(ServeClientBase):
         # return "sample english"
         log_block("LLM translate input", f"{text}")
         start_time = time.perf_counter()
-        ret =  translate_pipes.translate(text, self.language, self.dst_lang)
         translated_text = ret.translate_content
         log_block("LLM translate time", f"{(time.perf_counter() - start_time):.3f}", "s")
         log_block("LLM translate out", f"{translated_text}")
@@ -227,12 +220,12 @@ class PyWhiperCppServe(ServeClientBase):
             )
     def send_to_client(self, data:TransResult):
-        try:
-            self.websocket.send(
-               Message(result=data, request_id=self.client_uid).model_dump_json(by_alias=True)
-            )
-        except Exception as e:
-            logger.error(f"Sending data to client: {e}")
     def get_audio_chunk_for_processing(self):
         if self.frames_np.shape[0] >= self.sample_rate * 1:

 import numpy as np
 from logging import getLogger
+import asyncio
 from .utils import save_to_wave
 import time
 import json
 logger = getLogger("TranslatorApp")
 class PyWhiperCppServe(ServeClientBase):
+    def __init__(self, websocket, pipe:TranslatePipes,language=None, dst_lang=None, client_uid=None,):
         super().__init__(client_uid, websocket)
         self.language = language
         self.dst_lang = dst_lang # 目标翻译语言
         self._text_buffer = TripleTextBuffer()
         # 存储转录数据
         self._segment_manager = SegmentManager()
+        self._translate_pipes = pipe
         self.lock = threading.Lock()
         self.frames_np = None
         self._frame_queue = queue.Queue()
     def vad_merge(self):
         with self.lock:
             frame = self.frames_np.copy()
+            item = self._translate_pipes.voice_detect(frame.tobytes())
             if item.audio != b'':
                 frame_np = np.frombuffer(item.audio, dtype=np.float32)
                 self.frames_np = frame_np.copy()
         log_block("Audio buffer length", f"{audio_buffer.shape[0]/self.sample_rate:.2f}", "s")
         start_time = time.perf_counter()
+        item = self._translate_pipes.transcrible(audio_buffer.tobytes(), self.language)
         segments = item.segments
         log_block("Whisper transcrible time", f"{(time.perf_counter() - start_time):.3f}", "s")
         # return "sample english"
         log_block("LLM translate input", f"{text}")
         start_time = time.perf_counter()
+        ret =  self._translate_pipes.translate(text, self.language, self.dst_lang)
         translated_text = ret.translate_content
         log_block("LLM translate time", f"{(time.perf_counter() - start_time):.3f}", "s")
         log_block("LLM translate out", f"{translated_text}")
             )
     def send_to_client(self, data:TransResult):
+        coro = self.websocket.send_text(
+            Message(result=data, request_id=self.client_uid).model_dump_json(by_alias=True)
+        )
+        asyncio.run(coro)
     def get_audio_chunk_for_processing(self):
         if self.frames_np.shape[0] >= self.sample_rate * 1:

uv.lock CHANGED Viewed

@@ -207,6 +207,18 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/0e/f6/65ecc6878a89bb1c23a086ea335ad4bf21a588990c3f535a227b9eea9108/charset_normalizer-3.4.1-py3-none-any.whl", hash = "sha256:d98b1668f06378c6dbefec3b92299716b931cd4e6061f3c875a71ced1780ab85", size = 49767 },
 ]
 [[package]]
 name = "colorama"
 version = "0.4.6"
@@ -278,6 +290,15 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/5c/0d/24d40adaacf77f133ac87a29c045ee0d81bb99732b09b6ff0251c76e5c67/fsspec-2025.3.1-py3-none-any.whl", hash = "sha256:2ce85886f37dfa12d5ad4764f1342efbf00ec0a4fe164f070038499d80142887", size = 194444 },
 ]
 [[package]]
 name = "humanfriendly"
 version = "10.0"
@@ -1205,6 +1226,7 @@ dependencies = [
     { name = "soundfile" },
     { name = "torch" },
     { name = "tqdm" },
     { name = "websocket-client" },
     { name = "websockets" },
 ]
@@ -1224,6 +1246,7 @@ requires-dist = [
     { name = "soundfile", specifier = ">=0.13.1" },
     { name = "torch", specifier = ">=2.6.0" },
     { name = "tqdm", specifier = ">=4.67.1" },
     { name = "websocket-client", specifier = ">=1.8.0" },
     { name = "websockets", specifier = ">=15.0.1" },
 ]
@@ -1268,6 +1291,19 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/c8/19/4ec628951a74043532ca2cf5d97b7b14863931476d117c471e8e2b1eb39f/urllib3-2.3.0-py3-none-any.whl", hash = "sha256:1cee9ad369867bfdbbb48b7dd50374c0967a0bb7710050facf0dd6911440e3df", size = 128369 },
 ]
 [[package]]
 name = "websocket-client"
 version = "1.8.0"

     { url = "https://files.pythonhosted.org/packages/0e/f6/65ecc6878a89bb1c23a086ea335ad4bf21a588990c3f535a227b9eea9108/charset_normalizer-3.4.1-py3-none-any.whl", hash = "sha256:d98b1668f06378c6dbefec3b92299716b931cd4e6061f3c875a71ced1780ab85", size = 49767 },
 ]
+[[package]]
+name = "click"
+version = "8.1.8"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "colorama", marker = "sys_platform == 'win32'" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/b9/2e/0090cbf739cee7d23781ad4b89a9894a41538e4fcf4c31dcdd705b78eb8b/click-8.1.8.tar.gz", hash = "sha256:ed53c9d8990d83c2a27deae68e4ee337473f6330c040a31d4225c9574d16096a", size = 226593 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/7e/d4/7ebdbd03970677812aac39c869717059dbb71a4cfc033ca6e5221787892c/click-8.1.8-py3-none-any.whl", hash = "sha256:63c132bbbed01578a06712a2d1f497bb62d9c1c0d329b7903a866228027263b2", size = 98188 },
+]
 [[package]]
 name = "colorama"
 version = "0.4.6"
     { url = "https://files.pythonhosted.org/packages/5c/0d/24d40adaacf77f133ac87a29c045ee0d81bb99732b09b6ff0251c76e5c67/fsspec-2025.3.1-py3-none-any.whl", hash = "sha256:2ce85886f37dfa12d5ad4764f1342efbf00ec0a4fe164f070038499d80142887", size = 194444 },
 ]
+[[package]]
+name = "h11"
+version = "0.14.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/f5/38/3af3d3633a34a3316095b39c8e8fb4853a28a536e55d347bd8d8e9a14b03/h11-0.14.0.tar.gz", hash = "sha256:8f19fbbe99e72420ff35c00b27a34cb9937e902a8b810e2c88300c6f0a3b699d", size = 100418 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/95/04/ff642e65ad6b90db43e668d70ffb6736436c7ce41fcc549f4e9472234127/h11-0.14.0-py3-none-any.whl", hash = "sha256:e3fe4ac4b851c468cc8363d500db52c2ead036020723024a109d37346efaa761", size = 58259 },
+]
 [[package]]
 name = "humanfriendly"
 version = "10.0"
     { name = "soundfile" },
     { name = "torch" },
     { name = "tqdm" },
+    { name = "uvicorn" },
     { name = "websocket-client" },
     { name = "websockets" },
 ]
     { name = "soundfile", specifier = ">=0.13.1" },
     { name = "torch", specifier = ">=2.6.0" },
     { name = "tqdm", specifier = ">=4.67.1" },
+    { name = "uvicorn", specifier = ">=0.34.0" },
     { name = "websocket-client", specifier = ">=1.8.0" },
     { name = "websockets", specifier = ">=15.0.1" },
 ]
     { url = "https://files.pythonhosted.org/packages/c8/19/4ec628951a74043532ca2cf5d97b7b14863931476d117c471e8e2b1eb39f/urllib3-2.3.0-py3-none-any.whl", hash = "sha256:1cee9ad369867bfdbbb48b7dd50374c0967a0bb7710050facf0dd6911440e3df", size = 128369 },
 ]
+[[package]]
+name = "uvicorn"
+version = "0.34.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "click" },
+    { name = "h11" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/4b/4d/938bd85e5bf2edeec766267a5015ad969730bb91e31b44021dfe8b22df6c/uvicorn-0.34.0.tar.gz", hash = "sha256:404051050cd7e905de2c9a7e61790943440b3416f49cb409f965d9dcd0fa73e9", size = 76568 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/61/14/33a3a1352cfa71812a3a21e8c9bfb83f60b0011f5e36f2b1399d51928209/uvicorn-0.34.0-py3-none-any.whl", hash = "sha256:023dc038422502fa28a09c7a30bf2b6991512da7dcdb8fd35fe57cfc154126f4", size = 62315 },
+]
 [[package]]
 name = "websocket-client"
 version = "1.8.0"