Upload 11 files

app.js

@@ -0,0 +1,99 @@
+const demoapp = {
+    text: '讲个冷笑话吧，要很好笑的那种。',
+    recording: false,
+    asrWS: null,
+    currentText: null,
+    disabled: false,
+    elapsedTime: null,
+    logs: [{ idx: 0, text: 'Happily here at ruzhila.cn.' }],
+    async init() {
+    },
+    async dotts() {
+        let audioContext = new AudioContext({ sampleRate: 16000 })
+        await audioContext.audioWorklet.addModule('./audio_process.js')
+
+        const ws = new WebSocket('/tts');
+        ws.onopen = () => {
+            ws.send(this.text);
+        };
+        const playNode = new AudioWorkletNode(audioContext, 'play-audio-processor');
+        playNode.connect(audioContext.destination);
+
+        this.disabled = true;
+        ws.onmessage = async (e) => {
+            if (e.data instanceof Blob) {
+                e.data.arrayBuffer().then((arrayBuffer) => {
+                    const int16Array = new Int16Array(arrayBuffer);
+                    let float32Array = new Float32Array(int16Array.length);
+                    for (let i = 0; i < int16Array.length; i++) {
+                        float32Array[i] = int16Array[i] / 32768.;
+                    }
+                    playNode.port.postMessage({ message: 'audioData', audioData: float32Array });
+                });
+            } else {
+                this.elapsedTime = JSON.parse(e.data)?.elapsed;
+                this.disabled = false;
+            }
+        }
+    },
+
+    async stopasr() {
+        if (!this.asrWS) {
+            return;
+        }
+        this.asrWS.close();
+        this.asrWS = null;
+        this.recording = false;
+        if (this.currentText) {
+            this.logs.push({ idx: this.logs.length + 1, text: this.currentText });
+        }
+        this.currentText = null;
+
+    },
+
+    async doasr() {
+        const audioConstraints = {
+            video: false,
+            audio: true,
+        };
+
+        const mediaStream = await navigator.mediaDevices.getUserMedia(audioConstraints);
+
+        const ws = new WebSocket('/asr');
+        let currentMessage = '';
+
+        ws.onopen = () => {
+            this.logs = [];
+        };
+
+        ws.onmessage = (e) => {
+            const data = JSON.parse(e.data);
+            const { text, finished, idx } = data;
+
+            currentMessage = text;
+            this.currentText = text
+
+            if (finished) {
+                this.logs.push({ text: currentMessage, idx: idx });
+                currentMessage = '';
+                this.currentText = null
+            }
+        };
+
+        let audioContext = new AudioContext({ sampleRate: 16000 })
+        await audioContext.audioWorklet.addModule('./audio_process.js')
+
+        const recordNode = new AudioWorkletNode(audioContext, 'record-audio-processor');
+        recordNode.connect(audioContext.destination);
+        recordNode.port.onmessage = (event) => {
+            if (ws && ws.readyState === WebSocket.OPEN) {
+                const int16Array = event.data.data;
+                ws.send(int16Array.buffer);
+            }
+        }
+        const source = audioContext.createMediaStreamSource(mediaStream);
+        source.connect(recordNode);
+        this.asrWS = ws;
+        this.recording = true;
+    }
+}

app.py

@@ -0,0 +1,194 @@
+from typing import *
+from fastapi import FastAPI, HTTPException, Request, WebSocket, WebSocketDisconnect, Query
+from fastapi.responses import HTMLResponse, StreamingResponse
+from fastapi.staticfiles import StaticFiles
+import asyncio
+import logging
+from pydantic import BaseModel, Field
+import uvicorn
+from voiceapi.tts import TTSResult, start_tts_stream, TTSStream
+from voiceapi.asr import start_asr_stream, ASRStream, ASRResult
+import logging
+import argparse
+import os
+
+app = FastAPI()
+logger = logging.getLogger(__file__)
+
+
+@app.websocket("/asr")
+async def websocket_asr(websocket: WebSocket,
+                        samplerate: int = Query(16000, title="Sample Rate",
+                                                description="The sample rate of the audio."),):
+    await websocket.accept()
+
+    asr_stream: ASRStream = await start_asr_stream(samplerate, args)
+    if not asr_stream:
+        logger.error("failed to start ASR stream")
+        await websocket.close()
+        return
+
+    async def task_recv_pcm():
+        while True:
+            pcm_bytes = await websocket.receive_bytes()
+            if not pcm_bytes:
+                return
+            await asr_stream.write(pcm_bytes)
+
+    async def task_send_result():
+        while True:
+            result: ASRResult = await asr_stream.read()
+            if not result:
+                return
+            await websocket.send_json(result.to_dict())
+    try:
+        await asyncio.gather(task_recv_pcm(), task_send_result())
+    except WebSocketDisconnect:
+        logger.info("asr: disconnected")
+    finally:
+        await asr_stream.close()
+
+
+@app.websocket("/tts")
+async def websocket_tts(websocket: WebSocket,
+                        samplerate: int = Query(16000,
+                                                title="Sample Rate",
+                                                description="The sample rate of the generated audio."),
+                        interrupt: bool = Query(True,
+                                                title="Interrupt",
+                                                description="Interrupt the current TTS stream when a new text is received."),
+                        sid: int = Query(0,
+                                         title="Speaker ID",
+                                         description="The ID of the speaker to use for TTS."),
+                        chunk_size: int = Query(1024,
+                                                title="Chunk Size",
+                                                description="The size of the chunk to send to the client."),
+                        speed: float = Query(1.0,
+                                             title="Speed",
+                                             description="The speed of the generated audio."),
+                        split: bool = Query(True,
+                                            title="Split",
+                                            description="Split the text into sentences.")):
+
+    await websocket.accept()
+    tts_stream: TTSStream = None
+
+    async def task_recv_text():
+        nonlocal tts_stream
+        while True:
+            text = await websocket.receive_text()
+            if not text:
+                return
+
+            if interrupt or not tts_stream:
+                if tts_stream:
+                    await tts_stream.close()
+                    logger.info("tts: stream interrupt")
+
+                tts_stream = await start_tts_stream(sid, samplerate, speed, args)
+                if not tts_stream:
+                    logger.error("tts: failed to allocate tts stream")
+                    await websocket.close()
+                    return
+            logger.info(f"tts: received: {text} (split={split})")
+            await tts_stream.write(text, split)
+
+    async def task_send_pcm():
+        nonlocal tts_stream
+        while not tts_stream:
+            # wait for tts stream to be created
+            await asyncio.sleep(0.1)
+
+        while True:
+            result: TTSResult = await tts_stream.read()
+            if not result:
+                return
+
+            if result.finished:
+                await websocket.send_json(result.to_dict())
+            else:
+                for i in range(0, len(result.pcm_bytes), chunk_size):
+                    await websocket.send_bytes(result.pcm_bytes[i:i+chunk_size])
+
+    try:
+        await asyncio.gather(task_recv_text(), task_send_pcm())
+    except WebSocketDisconnect:
+        logger.info("tts: disconnected")
+    finally:
+        if tts_stream:
+            await tts_stream.close()
+
+
+class TTSRequest(BaseModel):
+    text: str = Field(..., title="Text",
+                      description="The text to be converted to speech.",
+                      examples=["Hello, world!"])
+    sid: int = Field(0, title="Speaker ID",
+                     description="The ID of the speaker to use for TTS.")
+    samplerate: int = Field(16000, title="Sample Rate",
+                            description="The sample rate of the generated audio.")
+    speed: float = Field(1.0, title="Speed",
+                         description="The speed of the generated audio.")
+
+
+@ app.post("/tts",
+           description="Generate speech audio from text.",
+           response_class=StreamingResponse, responses={200: {"content": {"audio/wav": {}}}})
+async def tts_generate(req: TTSRequest):
+    if not req.text:
+        raise HTTPException(status_code=400, detail="text is required")
+
+    tts_stream = await start_tts_stream(req.sid, req.samplerate, req.speed,  args)
+    if not tts_stream:
+        raise HTTPException(
+            status_code=500, detail="failed to start TTS stream")
+
+    r = await tts_stream.generate(req.text)
+    return StreamingResponse(r, media_type="audio/wav")
+
+
+if __name__ == "__main__":
+    models_root = './models'
+
+    for d in ['.', '..', '../..']:
+        if os.path.isdir(f'{d}/models'):
+            models_root = f'{d}/models'
+            break
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--port", type=int, default=8000, help="port number")
+    parser.add_argument("--addr", type=str,
+                        default="0.0.0.0", help="serve address")
+
+    parser.add_argument("--asr-provider", type=str,
+                        default="cpu", help="asr provider, cpu or cuda")
+    parser.add_argument("--tts-provider", type=str,
+                        default="cpu", help="tts provider, cpu or cuda")
+
+    parser.add_argument("--threads", type=int, default=2,
+                        help="number of threads")
+
+    parser.add_argument("--models-root", type=str, default=models_root,
+                        help="model root directory")
+
+    parser.add_argument("--asr-model", type=str, default='sensevoice',
+                        help="ASR model name: zipformer-bilingual, sensevoice, paraformer-trilingual, paraformer-en")
+
+    parser.add_argument("--asr-lang", type=str, default='zh',
+                        help="ASR language, zh, en, ja, ko, yue")
+
+    parser.add_argument("--tts-model", type=str, default='vits-zh-hf-theresa',
+                        help="TTS model name: vits-zh-hf-theresa, vits-melo-tts-zh_en")
+
+    args = parser.parse_args()
+
+    if args.tts_model == 'vits-melo-tts-zh_en' and args.tts_provider == 'cuda':
+        logger.warning(
+            "vits-melo-tts-zh_en does not support CUDA fallback to CPU")
+        args.tts_provider = 'cpu'
+
+    app.mount("/", app=StaticFiles(directory="./assets", html=True), name="assets")
+
+    logging.basicConfig(format='%(levelname)s: %(asctime)s %(name)s:%(lineno)s %(message)s',
+                        level=logging.INFO)
+    uvicorn.run(app, host=args.addr, port=args.port)

asr.py

@@ -0,0 +1,233 @@
+from typing import *
+import logging
+import time
+import logging
+import sherpa_onnx
+import os
+import asyncio
+import numpy as np
+
+logger = logging.getLogger(__file__)
+_asr_engines = {}
+
+
+class ASRResult:
+    def __init__(self, text: str, finished: bool, idx: int):
+        self.text = text
+        self.finished = finished
+        self.idx = idx
+
+    def to_dict(self):
+        return {"text": self.text, "finished": self.finished, "idx": self.idx}
+
+
+class ASRStream:
+    def __init__(self, recognizer: Union[sherpa_onnx.OnlineRecognizer | sherpa_onnx.OfflineRecognizer], sample_rate: int) -> None:
+        self.recognizer = recognizer
+        self.inbuf = asyncio.Queue()
+        self.outbuf = asyncio.Queue()
+        self.sample_rate = sample_rate
+        self.is_closed = False
+        self.online = isinstance(recognizer, sherpa_onnx.OnlineRecognizer)
+
+    async def start(self):
+        if self.online:
+            asyncio.create_task(self.run_online())
+        else:
+            asyncio.create_task(self.run_offline())
+
+    async def run_online(self):
+        stream = self.recognizer.create_stream()
+        last_result = ""
+        segment_id = 0
+        logger.info('asr: start real-time recognizer')
+        while not self.is_closed:
+            samples = await self.inbuf.get()
+            stream.accept_waveform(self.sample_rate, samples)
+            while self.recognizer.is_ready(stream):
+                self.recognizer.decode_stream(stream)
+
+            is_endpoint = self.recognizer.is_endpoint(stream)
+            result = self.recognizer.get_result(stream)
+
+            if result and (last_result != result):
+                last_result = result
+                logger.info(f' > {segment_id}:{result}')
+                self.outbuf.put_nowait(
+                    ASRResult(result, False, segment_id))
+
+            if is_endpoint:
+                if result:
+                    logger.info(f'{segment_id}: {result}')
+                    self.outbuf.put_nowait(
+                        ASRResult(result, True, segment_id))
+                    segment_id += 1
+                self.recognizer.reset(stream)
+
+    async def run_offline(self):
+        vad = _asr_engines['vad']
+        segment_id = 0
+        st = None
+        while not self.is_closed:
+            samples = await self.inbuf.get()
+            vad.accept_waveform(samples)
+            while not vad.empty():
+                if not st:
+                    st = time.time()
+                stream = self.recognizer.create_stream()
+                stream.accept_waveform(self.sample_rate, vad.front.samples)
+
+                vad.pop()
+                self.recognizer.decode_stream(stream)
+
+                result = stream.result.text.strip()
+                if result:
+                    duration = time.time() - st
+                    logger.info(f'{segment_id}:{result} ({duration:.2f}s)')
+                    self.outbuf.put_nowait(ASRResult(result, True, segment_id))
+                    segment_id += 1
+            st = None
+
+    async def close(self):
+        self.is_closed = True
+        self.outbuf.put_nowait(None)
+
+    async def write(self, pcm_bytes: bytes):
+        pcm_data = np.frombuffer(pcm_bytes, dtype=np.int16)
+        samples = pcm_data.astype(np.float32) / 32768.0
+        self.inbuf.put_nowait(samples)
+
+    async def read(self) -> ASRResult:
+        return await self.outbuf.get()
+
+
+def create_zipformer(samplerate: int, args) -> sherpa_onnx.OnlineRecognizer:
+    d = os.path.join(
+        args.models_root, 'sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20')
+    if not os.path.exists(d):
+        raise ValueError(f"asr: model not found {d}")
+
+    encoder = os.path.join(d, "encoder-epoch-99-avg-1.onnx")
+    decoder = os.path.join(d, "decoder-epoch-99-avg-1.onnx")
+    joiner = os.path.join(d, "joiner-epoch-99-avg-1.onnx")
+    tokens = os.path.join(d, "tokens.txt")
+
+    recognizer = sherpa_onnx.OnlineRecognizer.from_transducer(
+        tokens=tokens,
+        encoder=encoder,
+        decoder=decoder,
+        joiner=joiner,
+        provider=args.asr_provider,
+        num_threads=args.threads,
+        sample_rate=samplerate,
+        feature_dim=80,
+        enable_endpoint_detection=True,
+        rule1_min_trailing_silence=2.4,
+        rule2_min_trailing_silence=1.2,
+        rule3_min_utterance_length=20,  # it essentially disables this rule
+    )
+    return recognizer
+
+
+def create_sensevoice(samplerate: int, args) -> sherpa_onnx.OfflineRecognizer:
+    d = os.path.join(args.models_root,
+                     'sherpa-onnx-sense-voice-zh-en-ja-ko-yue-2024-07-17')
+
+    if not os.path.exists(d):
+        raise ValueError(f"asr: model not found {d}")
+
+    recognizer = sherpa_onnx.OfflineRecognizer.from_sense_voice(
+        model=os.path.join(d, 'model.onnx'),
+        tokens=os.path.join(d, 'tokens.txt'),
+        num_threads=args.threads,
+        sample_rate=samplerate,
+        use_itn=True,
+        debug=0,
+        language=args.asr_lang,
+    )
+    return recognizer
+
+
+def create_paraformer_trilingual(samplerate: int, args) -> sherpa_onnx.OnlineRecognizer:
+    d = os.path.join(
+        args.models_root, 'sherpa-onnx-paraformer-trilingual-zh-cantonese-en')
+    if not os.path.exists(d):
+        raise ValueError(f"asr: model not found {d}")
+
+    recognizer = sherpa_onnx.OfflineRecognizer.from_paraformer(
+        paraformer=os.path.join(d, 'model.onnx'),
+        tokens=os.path.join(d, 'tokens.txt'),
+        num_threads=args.threads,
+        sample_rate=samplerate,
+        debug=0,
+        provider=args.asr_provider,
+    )
+    return recognizer
+
+
+def create_paraformer_en(samplerate: int, args) -> sherpa_onnx.OnlineRecognizer:
+    d = os.path.join(
+        args.models_root, 'sherpa-onnx-paraformer-en')
+    if not os.path.exists(d):
+        raise ValueError(f"asr: model not found {d}")
+
+    recognizer = sherpa_onnx.OfflineRecognizer.from_paraformer(
+        paraformer=os.path.join(d, 'model.onnx'),
+        tokens=os.path.join(d, 'tokens.txt'),
+        num_threads=args.threads,
+        sample_rate=samplerate,
+        use_itn=True,
+        debug=0,
+        provider=args.asr_provider,
+    )
+    return recognizer
+
+
+def load_asr_engine(samplerate: int, args) -> sherpa_onnx.OnlineRecognizer:
+    cache_engine = _asr_engines.get(args.asr_model)
+    if cache_engine:
+        return cache_engine
+    st = time.time()
+    if args.asr_model == 'zipformer-bilingual':
+        cache_engine = create_zipformer(samplerate, args)
+    elif args.asr_model == 'sensevoice':
+        cache_engine = create_sensevoice(samplerate, args)
+        _asr_engines['vad'] = load_vad_engine(samplerate, args)
+    elif args.asr_model == 'paraformer-trilingual':
+        cache_engine = create_paraformer_trilingual(samplerate, args)
+        _asr_engines['vad'] = load_vad_engine(samplerate, args)
+    elif args.asr_model == 'paraformer-en':
+        cache_engine = create_paraformer_en(samplerate, args)
+        _asr_engines['vad'] = load_vad_engine(samplerate, args)
+    else:
+        raise ValueError(f"asr: unknown model {args.asr_model}")
+    _asr_engines[args.asr_model] = cache_engine
+    logger.info(f"asr: engine loaded in {time.time() - st:.2f}s")
+    return cache_engine
+
+
+def load_vad_engine(samplerate: int, args, min_silence_duration: float = 0.25, buffer_size_in_seconds: int = 100) -> sherpa_onnx.VoiceActivityDetector:
+    config = sherpa_onnx.VadModelConfig()
+    d = os.path.join(args.models_root, 'silero_vad')
+    if not os.path.exists(d):
+        raise ValueError(f"vad: model not found {d}")
+
+    config.silero_vad.model = os.path.join(d, 'silero_vad.onnx')
+    config.silero_vad.min_silence_duration = min_silence_duration
+    config.sample_rate = samplerate
+    config.provider = args.asr_provider
+    config.num_threads = args.threads
+
+    vad = sherpa_onnx.VoiceActivityDetector(
+        config,
+        buffer_size_in_seconds=buffer_size_in_seconds)
+    return vad
+
+
+async def start_asr_stream(samplerate: int, args) -> ASRStream:
+    """
+    Start a ASR stream
+    """
+    stream = ASRStream(load_asr_engine(samplerate, args), samplerate)
+    await stream.start()
+    return stream

audio_process.js

@@ -0,0 +1,45 @@
+class PlayerAudioProcessor extends AudioWorkletProcessor {
+    constructor() {
+        super();
+        this.buffer = new Float32Array();
+        this.port.onmessage = (event) => {
+            let newFetchedData = new Float32Array(this.buffer.length + event.data.audioData.length);
+            newFetchedData.set(this.buffer, 0);
+            newFetchedData.set(event.data.audioData, this.buffer.length);
+            this.buffer = newFetchedData;
+        };
+    }
+
+    process(inputs, outputs, parameters) {
+        const output = outputs[0];
+        const channel = output[0];
+        const bufferLength = this.buffer.length;
+        for (let i = 0; i < channel.length; i++) {
+            channel[i] = (i < bufferLength) ? this.buffer[i] : 0;
+        }
+        this.buffer = this.buffer.slice(channel.length);
+        return true;
+    }
+}
+
+class RecordAudioProcessor extends AudioWorkletProcessor {
+    constructor() {
+        super();
+    }
+
+    process(inputs, outputs, parameters) {
+        const channel = inputs[0][0];
+        if (!channel || channel.length === 0) {
+            return true;
+        }
+        const int16Array = new Int16Array(channel.length);
+        for (let i = 0; i < channel.length; i++) {
+            int16Array[i] = channel[i] * 32767;
+        }
+        this.port.postMessage({ data: int16Array });
+        return true
+    }
+}
+
+registerProcessor('play-audio-processor', PlayerAudioProcessor);
+registerProcessor('record-audio-processor', RecordAudioProcessor);

index.html

@@ -0,0 +1,179 @@
+<!DOCTYPE html>
+<html lang="en">
+
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <link rel="icon" type="image/svg+xml" href="./voice.png" />
+    <script src="//cdn.tailwindcss.com?plugins=forms"></script>
+    <link href="https://cdn.jsdelivr.net/npm/tailwindcss@latest/dist/tailwind.min.css" rel="stylesheet">
+    <script src="//cdn.jsdelivr.net/npm/[email protected]/dist/cdn.min.js" defer></script>
+    <script src="./app.js"></script>
+    <title>voiceapi demo </title>
+    <style>
+        * {
+            margin: 0;
+            padding: 0;
+        }
+    </style>
+
+    <style type="text/tailwindcss">
+        .label { @apply text-gray-900 w-[50px] lg:w-20 }
+        .title{
+            @apply text-[16px] text-zinc-500 mx-2;
+        }
+
+        .select { @apply w-full rounded-md h-10 }
+
+        .round { @apply rounded border px-3 p-2 border-slate-300 placeholder-gray-400 placeholder:text-sm
+                        focus:bg-white focus:text-gray-900 focus:placeholder-gray-500 focus:outline-none
+                         focus:border-zinc-950 focus:border ring-0 focus:ring-0 text-gray-900 }
+
+        .checkbox { @apply ml-2 lg:ml-4 border focus:outline-none ring-0 focus:ring-gray-800 text-gray-900 }
+        .dash{ @apply border border-dashed border-zinc-200 flex flex-grow }
+
+        .button { @apply hover:bg-opacity-90 text-white font-bold py-1.5 px-6 rounded-full cursor-pointer }
+        .card { @apply bg-white shadow-sm rounded-xl border p-4 }
+
+
+    .animate-ping { 
+        animation: ping 2s cubic-bezier(0.5, 0.4, 0.2, 1) infinite;
+    }
+
+    @keyframes ping {
+        0% {
+            transform: scale(1);
+            opacity: 1;
+        }
+        50% {
+            transform: scale(1.2);
+            opacity: 0.7;
+        }
+        100% {
+            transform: scale(1);
+            opacity: 1;
+        }
+    }
+    </style>
+</head>
+
+<body>
+    <script>
+        async function initAudioWorklet() {
+            try {
+                // Check for browser support
+                if (!('AudioContext' in window) || !('audioWorklet' in AudioContext.prototype)) {
+                    console.error('Audio Worklet API is not supported in this browser.');
+                    return;
+                }
+
+                // Initialize AudioContext
+                const audioContext = new AudioContext();
+
+                // Add Audio Worklet module
+                await audioContext.audioWorklet.addModule('./audio_process.js');
+
+                console.log('Audio Worklet module added successfully.');
+                // Your code to use the Audio Worklet goes here
+
+            } catch (error) {
+                console.error('Error initializing Audio Worklet:', error);
+            }
+        }
+
+        // Initialize Audio Worklet when the page is loaded
+        window.addEventListener('load', initAudioWorklet);
+    </script>
+    <div x-data="demoapp">
+        <header class="bg-gray-900 py-4 px-5 lg:p-4 lg:px-10 text-white sticky top-0 z-20">
+            <div class="flex w-full justify-between items-center">
+                <p class="gap-x-3">
+                    <span>VoiceAPI Demo</span> /
+                    <a href="https://ruzhila.cn/?from=voiceapi_demo">ruzhila.cn</a>
+                </p>
+                <a target="_blank" href="https://github.com/ruzhila/voiceapi" class="hover:cursor-pointer">
+                    <svg t="1724996252746" class="icon" viewBox="0 0 1024 1024" version="1.1"
+                        xmlns="http://www.w3.org/2000/svg" p-id="" width="25" height="25">
+                        <path
+                            d="M512 12.64c-282.752 0-512 229.216-512 512 0 226.208 146.72 418.144 350.144 485.824 25.6 4.736 35.008-11.104 35.008-24.64 0-12.192-0.48-52.544-0.704-95.328-142.464 30.976-172.512-60.416-172.512-60.416-23.296-59.168-56.832-74.912-56.832-74.912-46.464-31.776 3.52-31.136 3.52-31.136 51.392 3.616 78.464 52.768 78.464 52.768 45.664 78.272 119.776 55.648 148.992 42.56 4.576-33.088 17.856-55.68 32.512-68.48-113.728-12.928-233.28-56.864-233.28-253.024 0-55.904 20-101.568 52.768-137.44-5.312-12.896-22.848-64.96 4.96-135.488 0 0 43.008-13.76 140.832 52.48 40.832-11.36 84.64-17.024 128.16-17.248 43.488 0.192 87.328 5.888 128.256 17.248 97.728-66.24 140.64-52.48 140.64-52.48 27.872 70.528 10.336 122.592 5.024 135.488 32.832 35.84 52.704 81.536 52.704 137.44 0 196.64-119.776 239.936-233.792 252.64 18.368 15.904 34.72 47.04 34.72 94.816 0 68.512-0.608 123.648-0.608 140.512 0 13.632 9.216 29.6 35.168 24.576 203.328-67.776 349.856-259.616 349.856-485.76 0-282.784-229.248-512-512-512z"
+                            fill="#ffffff"></path>
+                    </svg>
+                </a>
+            </div>
+        </header>
+
+        <div class="flex px-6 gap-x-10 w-full max-w-7xl mx-auto">
+            <div class="relative flex flex-col items-center w-1/3 py-10">
+                <div class="w-full">
+                    <textarea x-model="text" class="round p-4 w-full h-[36rem] text-sm"
+                        placeholder="Enter text here"></textarea>
+                </div>
+
+                <div>
+                    <button @click="dotts" :disabled="disabled"
+                        class="button bg-gray-900 flex items-center gap-x-2 mt-6">
+                        <span>Speak</span>
+                        <svg t="1726215464577" class="icon" viewBox="0 0 1024 1024" version="1.1"
+                            xmlns="http://www.w3.org/2000/svg" p-id="4263" width="20" height="20">
+                            <path
+                                d="M830.450526 853.759999q-11.722105 8.791579-27.351579 8.791579-19.536842 0-33.701053-14.164211t-14.164211-33.701053q0-21.490526 
+                   16.606316-36.143158 0.976842-0.976842 1.953684-1.465263t1.953684-1.465263l0.976842-0.976842q27.351579-18.56 50.795789-43.957895t41.027368-55.191579 27.351579-63.494737 9.768421-69.84421q0-73.263158-37.12-133.827368t-92.8-99.637895q-20.513684-14.652632-20.513684-39.073684 0-19.536842 14.164211-33.701053t33.701053-14.164211q16.606316 0 29.305263 10.745263 36.143158 25.397895 67.402105 59.098947t53.726316 73.263158 35.166316 84.496842 12.698947 92.8q0 48.842105-12.698947 93.776842t-35.654737 84.985263-54.214737 73.751579-68.378947 59.098947zM775.747368 415.157894q20.513684 28.328421 32.72421 57.145263t12.210526 69.84421q0 39.073684-12.698947 70.332632t-32.235789 56.656842q-7.814737 10.745263-16.606316 19.048421t-22.467368 8.303158q-17.583158 0-29.793684-12.698947t-12.210526-30.282105q0-7.814737 2.930526-15.629474l-0.976842 0q4.884211-10.745263 11.722105-20.513684t13.187368-20.025263 10.745263-23.444211 4.395789-31.747368q0-17.583158-4.395789-30.770526t-10.745263-23.932632-13.187368-20.513684-10.745263-20.513684q-2.930526-6.837895-2.930526-15.629474 0-17.583158 12.210526-30.282105t29.793684-12.698947q13.675789 0 22.467368 8.303158t16.606316 19.048421zM460.227368 995.402104q-49.818947-44.934737-105.498947-93.776842t-103.545263-89.869474q-55.68-46.888421-111.36-92.8-10.745263 0.976842-21.490526 0.976842-8.791579 0.976842-18.56 0.976842l-16.606316 0q-26.374737 0-42.981053-16.117895t-16.606316-38.585263l0-246.16421 0.976842 0-0.976842-0.976842q0-27.351579 17.094737-44.934737t42.492632-17.583158l55.68 0q89.869474-76.193684 163.132631-136.757895 31.258947-26.374737 61.541053-51.28421t54.703158-45.423158 41.027368-34.189474 20.513684-16.606316q29.305263-21.490526 47.376842-19.536842t28.328421 17.583158 14.164211 38.096842 3.907368 41.027368l0 788.311578 0 2.930526q0 18.56-6.837895 39.562105t-21.002105 33.212632-35.654737 10.256842-49.818947-28.328421z"
+                                p-id="4264" fill="#ffffff"></path>
+                        </svg>
+                    </button>
+                </div>
+                <template x-if="elapsedTime">
+                    <p x-text="`elapsedTime: ${elapsedTime}`" class="mt-4 text-sm text-gray-600 "></p>
+                </template>
+            </div>
+
+            <!-- recording  -->
+            <div class="w-full flex-grow h-[calc(100vh-10rem)] xl:pl-10 py-10">
+
+                <div
+                    class="rounded border border-gray-500 p-3 w-full flex flex-col items-end h-[36rem] overflow-y-auto">
+                    <template x-for="item in logs">
+                        <div class="mt-3 mb-2">
+                            <span
+                                class="text-white px-4 py-1.5 text-[13px] display-inline-block border border-gray-900 rounded-t-full rounded-l-full bg-gray-900 justify-end w-auto"
+                                x-text="item?.text">
+                            </span>
+                        </div>
+                    </template>
+                </div>
+
+
+                <template x-if="currentText">
+                    <p x-text="`${currentText} …`" class="text-gray-800 mt-4 text-sm text-center"></p>
+                </template>
+
+                <template x-if="!recording">
+                    <div class="flex flex-col gap-y-4 items-center justify-center mt-4">
+                        <p @click="doasr"
+                            class="mt-2 border border-gray-100 rounded-full duration-300 hover:scale-105 hover:border-gray-400">
+                            <img src="./images/record.svg" alt="" class="w-14 h-14 mx-auto cursor-pointer">
+                        </p>
+                        <p class="text-gray-600">Click to record !</p>
+                    </div>
+                </template>
+
+                <template x-if="recording">
+                    <div class="flex flex-col items-center justify-center gap-y-4 mt-4">
+
+                        <p @click="stopasr"
+                            class="mt-2 border border-red-100 rounded-full duration-300 hover:scale-105  hover:border-red-400">
+                            <img src="./images/speaking.svg" alt=""
+                                class="w-14 h-14 mx-auto cursor-pointer animate-ping">
+                        </p>
+                        <div class="flex items-center text-gray-600 gap-x-4">
+                            <p>Click to stop recording !</p>
+                        </div>
+                    </div>
+                </template>
+            </div>
+        </div>
+    </div>
+    </div>
+</body>
+
+</html>

requirements.txt

@@ -0,0 +1,7 @@
+sherpa-onnx == 1.10.24
+soundfile == 0.12.1
+fastapi == 0.114.1
+uvicorn == 0.30.6
+scipy  == 1.13.1
+numpy == 1.26.4
+websockets == 13.0.1

sherpa_examples.py

@@ -0,0 +1,274 @@
+#!/bin/env python3
+"""
+    Real-time ASR using microphone
+"""
+
+import argparse
+import logging
+import sherpa_onnx
+import os
+import time
+import struct
+import asyncio
+import soundfile
+
+try:
+    import pyaudio
+except ImportError:
+    raise ImportError('Please install pyaudio with `pip install pyaudio`')
+
+logger = logging.getLogger(__name__)
+SAMPLE_RATE = 16000
+
+pactx = pyaudio.PyAudio()
+models_root: str = None
+num_threads: int = 1
+
+
+def create_zipformer(args) -> sherpa_onnx.OnlineRecognizer:
+    d = os.path.join(
+        models_root, 'sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20')
+    encoder = os.path.join(d, "encoder-epoch-99-avg-1.onnx")
+    decoder = os.path.join(d, "decoder-epoch-99-avg-1.onnx")
+    joiner = os.path.join(d, "joiner-epoch-99-avg-1.onnx")
+    tokens = os.path.join(d, "tokens.txt")
+
+    recognizer = sherpa_onnx.OnlineRecognizer.from_transducer(
+        tokens=tokens,
+        encoder=encoder,
+        decoder=decoder,
+        joiner=joiner,
+        provider=args.provider,
+        num_threads=num_threads,
+        sample_rate=SAMPLE_RATE,
+        feature_dim=80,
+        enable_endpoint_detection=True,
+        rule1_min_trailing_silence=2.4,
+        rule2_min_trailing_silence=1.2,
+        rule3_min_utterance_length=20,  # it essentially disables this rule
+    )
+    return recognizer
+
+
+def create_sensevoice(args) -> sherpa_onnx.OfflineRecognizer:
+    model = os.path.join(
+        models_root, 'sherpa-onnx-sense-voice-zh-en-ja-ko-yue-2024-07-17', 'model.onnx')
+    tokens = os.path.join(
+        models_root, 'sherpa-onnx-sense-voice-zh-en-ja-ko-yue-2024-07-17', 'tokens.txt')
+    recognizer = sherpa_onnx.OfflineRecognizer.from_sense_voice(
+        model=model,
+        tokens=tokens,
+        num_threads=num_threads,
+        use_itn=True,
+        debug=0,
+        language=args.lang,
+    )
+    return recognizer
+
+
+async def run_online(buf, recognizer):
+    stream = recognizer.create_stream()
+    last_result = ""
+    segment_id = 0
+    logger.info('Start real-time recognizer')
+    while True:
+        samples = await buf.get()
+        stream.accept_waveform(SAMPLE_RATE, samples)
+        while recognizer.is_ready(stream):
+            recognizer.decode_stream(stream)
+
+        is_endpoint = recognizer.is_endpoint(stream)
+        result = recognizer.get_result(stream)
+
+        if result and (last_result != result):
+            last_result = result
+            logger.info(f' > {segment_id}:{result}')
+
+        if is_endpoint:
+            if result:
+                logger.info(f'{segment_id}: {result}')
+                segment_id += 1
+            recognizer.reset(stream)
+
+
+async def run_offline(buf, recognizer):
+    config = sherpa_onnx.VadModelConfig()
+    config.silero_vad.model = os.path.join(
+        models_root, 'silero_vad', 'silero_vad.onnx')
+    config.silero_vad.min_silence_duration = 0.25
+    config.sample_rate = SAMPLE_RATE
+    vad = sherpa_onnx.VoiceActivityDetector(
+        config, buffer_size_in_seconds=100)
+
+    logger.info('Start offline recognizer with VAD')
+    texts = []
+    while True:
+        samples = await buf.get()
+        vad.accept_waveform(samples)
+        while not vad.empty():
+            stream = recognizer.create_stream()
+            stream.accept_waveform(SAMPLE_RATE, vad.front.samples)
+
+            vad.pop()
+            recognizer.decode_stream(stream)
+
+            text = stream.result.text.strip().lower()
+            if len(text):
+                idx = len(texts)
+                texts.append(text)
+                logger.info(f"{idx}: {text}")
+
+
+async def handle_asr(args):
+    action_func = None
+    if args.model == 'zipformer':
+        recognizer = create_zipformer(args)
+        action_func = run_online
+    elif args.model == 'sensevoice':
+        recognizer = create_sensevoice(args)
+        action_func = run_offline
+    else:
+        raise ValueError(f'Unknown model: {args.model}')
+    buf = asyncio.Queue()
+    recorder_task = asyncio.create_task(run_record(buf))
+    asr_task = asyncio.create_task(action_func(buf, recognizer))
+    await asyncio.gather(asr_task, recorder_task)
+
+
+async def handle_tts(args):
+    model = os.path.join(
+        models_root, 'vits-melo-tts-zh_en', 'model.onnx')
+    lexicon = os.path.join(
+        models_root, 'vits-melo-tts-zh_en', 'lexicon.txt')
+    dict_dir = os.path.join(
+        models_root, 'vits-melo-tts-zh_en', 'dict')
+    tokens = os.path.join(
+        models_root, 'vits-melo-tts-zh_en', 'tokens.txt')
+    tts_config = sherpa_onnx.OfflineTtsConfig(
+        model=sherpa_onnx.OfflineTtsModelConfig(
+            vits=sherpa_onnx.OfflineTtsVitsModelConfig(
+                model=model,
+                lexicon=lexicon,
+                dict_dir=dict_dir,
+                tokens=tokens,
+            ),
+            provider=args.provider,
+            debug=0,
+            num_threads=num_threads,
+        ),
+        max_num_sentences=args.max_num_sentences,
+    )
+    if not tts_config.validate():
+        raise ValueError("Please check your config")
+
+    tts = sherpa_onnx.OfflineTts(tts_config)
+
+    start = time.time()
+    audio = tts.generate(args.text, sid=args.sid,
+                         speed=args.speed)
+    elapsed_seconds = time.time() - start
+    audio_duration = len(audio.samples) / audio.sample_rate
+    real_time_factor = elapsed_seconds / audio_duration
+
+    if args.output:
+        logger.info(f"Saved to {args.output}")
+        soundfile.write(
+            args.output,
+            audio.samples,
+            samplerate=audio.sample_rate,
+            subtype="PCM_16",
+        )
+
+    logger.info(f"The text is '{args.text}'")
+    logger.info(f"Elapsed seconds: {elapsed_seconds:.3f}")
+    logger.info(f"Audio duration in seconds: {audio_duration:.3f}")
+    logger.info(
+        f"RTF: {elapsed_seconds:.3f}/{audio_duration:.3f} = {real_time_factor:.3f}")
+
+
+async def run_record(buf: asyncio.Queue[list[float]]):
+    loop = asyncio.get_event_loop()
+
+    def on_input(in_data, frame_count, time_info, status):
+        samples = [
+            v/32768.0 for v in list(struct.unpack('<' + 'h' * frame_count, in_data))]
+        loop.create_task(buf.put(samples))
+        return (None, pyaudio.paContinue)
+
+    frame_size = 320
+    recorder = pactx.open(format=pyaudio.paInt16, channels=1,
+                          rate=SAMPLE_RATE, input=True,
+                          frames_per_buffer=frame_size,
+                          stream_callback=on_input)
+    recorder.start_stream()
+    logger.info('Start recording')
+
+    while recorder.is_active():
+        await asyncio.sleep(0.1)
+
+
+async def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--provider', default='cpu',
+                        help='onnxruntime provider, default is cpu, use cuda for GPU')
+
+    subparsers = parser.add_subparsers(help='commands help')
+
+    asr_parser = subparsers.add_parser('asr', help='run asr mode')
+    asr_parser.add_argument('--model', default='zipformer',
+                            help='model name, default is zipformer')
+    asr_parser.add_argument('--lang',  default='zh',
+                            help='language, default is zh')
+    asr_parser.set_defaults(func=handle_asr)
+
+    tts_parser = subparsers.add_parser('tts', help='run tts mode')
+    tts_parser.add_argument('--sid', type=int, default=0, help="""Speaker ID. Used only for multi-speaker models, e.g.
+        models trained using the VCTK dataset. Not used for single-speaker
+        models, e.g., models trained using the LJ speech dataset.
+        """)
+    tts_parser.add_argument('--output', type=str, default='output.wav',
+                            help='output file name, default is output.wav')
+    tts_parser.add_argument(
+        "--speed",
+        type=float,
+        default=1.0,
+        help="Speech speed. Larger->faster; smaller->slower",
+    )
+    tts_parser.add_argument(
+        "--max-num-sentences",
+        type=int,
+        default=2,
+        help="""Max number of sentences in a batch to avoid OOM if the input
+        text is very long. Set it to -1 to process all the sentences in a
+        single batch. A smaller value does not mean it is slower compared
+        to a larger one on CPU.
+        """,
+    )
+    tts_parser.add_argument(
+        "text",
+        type=str,
+        help="The input text to generate audio for",
+    )
+    tts_parser.set_defaults(func=handle_tts)
+
+    args = parser.parse_args()
+
+    if hasattr(args, 'func'):
+        await args.func(args)
+    else:
+        parser.print_help()
+
+if __name__ == '__main__':
+    logging.basicConfig(
+        format='%(levelname)s: %(asctime)s %(name)s:%(lineno)s %(message)s')
+    logging.getLogger().setLevel(logging.INFO)
+    painfo = pactx.get_default_input_device_info()
+    assert painfo['maxInputChannels'] >= 1, 'No input device'
+    logger.info('Default input device: %s', painfo['name'])
+
+    for d in ['.', '..', '../..']:
+        if os.path.isdir(f'{d}/models'):
+            models_root = f'{d}/models'
+            break
+    assert models_root, 'Could not find models directory'
+    asyncio.run(main())

tts.py

@@ -0,0 +1,216 @@
+from typing import *
+import os
+import time
+import sherpa_onnx
+import logging
+import numpy as np
+import asyncio
+import time
+import soundfile
+from scipy.signal import resample
+import io
+import re
+
+logger = logging.getLogger(__file__)
+
+splitter = re.compile(r'[,，。.!?！？;；、\n]')
+_tts_engines = {}
+
+tts_configs = {
+    'vits-zh-hf-theresa': {
+        'model': 'theresa.onnx',
+        'lexicon': 'lexicon.txt',
+        'dict_dir': 'dict',
+        'tokens': 'tokens.txt',
+        'sample_rate': 22050,
+        # 'rule_fsts': ['phone.fst', 'date.fst', 'number.fst'],
+    },
+    'vits-melo-tts-zh_en': {
+        'model': 'model.onnx',
+        'lexicon': 'lexicon.txt',
+        'dict_dir': 'dict',
+        'tokens': 'tokens.txt',
+        'sample_rate': 44100,
+        'rule_fsts': ['phone.fst', 'date.fst', 'number.fst'],
+    },
+}
+
+
+def load_tts_model(name: str, model_root: str, provider: str, num_threads: int = 1, max_num_sentences: int = 20) -> sherpa_onnx.OfflineTtsConfig:
+    cfg = tts_configs[name]
+    fsts = []
+    model_dir = os.path.join(model_root, name)
+    for f in cfg.get('rule_fsts', ''):
+        fsts.append(os.path.join(model_dir, f))
+    tts_rule_fsts = ','.join(fsts) if fsts else ''
+
+    model_config = sherpa_onnx.OfflineTtsModelConfig(
+        vits=sherpa_onnx.OfflineTtsVitsModelConfig(
+            model=os.path.join(model_dir, cfg['model']),
+            lexicon=os.path.join(model_dir, cfg['lexicon']),
+            dict_dir=os.path.join(model_dir, cfg['dict_dir']),
+            tokens=os.path.join(model_dir, cfg['tokens']),
+        ),
+        provider=provider,
+        debug=0,
+        num_threads=num_threads,
+    )
+    tts_config = sherpa_onnx.OfflineTtsConfig(
+        model=model_config,
+        rule_fsts=tts_rule_fsts,
+        max_num_sentences=max_num_sentences)
+
+    if not tts_config.validate():
+        raise ValueError("tts: invalid config")
+
+    return tts_config
+
+
+def get_tts_engine(args) -> Tuple[sherpa_onnx.OfflineTts, int]:
+    sample_rate = tts_configs[args.tts_model]['sample_rate']
+    cache_engine = _tts_engines.get(args.tts_model)
+    if cache_engine:
+        return cache_engine, sample_rate
+    st = time.time()
+    tts_config = load_tts_model(
+        args.tts_model, args.models_root, args.tts_provider)
+
+    cache_engine = sherpa_onnx.OfflineTts(tts_config)
+    elapsed = time.time() - st
+    logger.info(f"tts: loaded {args.tts_model} in {elapsed:.2f}s")
+    _tts_engines[args.tts_model] = cache_engine
+
+    return cache_engine, sample_rate
+
+
+class TTSResult:
+    def __init__(self, pcm_bytes: bytes, finished: bool):
+        self.pcm_bytes = pcm_bytes
+        self.finished = finished
+        self.progress: float = 0.0
+        self.elapsed: float = 0.0
+        self.audio_duration: float = 0.0
+        self.audio_size: int = 0
+
+    def to_dict(self):
+        return {
+            "progress": self.progress,
+            "elapsed": f'{int(self.elapsed * 1000)}ms',
+            "duration": f'{self.audio_duration:.2f}s',
+            "size": self.audio_size
+        }
+
+
+class TTSStream:
+    def __init__(self, engine, sid: int, speed: float = 1.0, sample_rate: int = 16000, original_sample_rate: int = 16000):
+        self.engine = engine
+        self.sid = sid
+        self.speed = speed
+        self.outbuf: asyncio.Queue[TTSResult | None] = asyncio.Queue()
+        self.is_closed = False
+        self.target_sample_rate = sample_rate
+        self.original_sample_rate = original_sample_rate
+
+    def on_process(self, chunk: np.ndarray, progress: float):
+        if self.is_closed:
+            return 0
+
+        # resample to target sample rate
+        if self.target_sample_rate != self.original_sample_rate:
+            num_samples = int(
+                len(chunk) * self.target_sample_rate / self.original_sample_rate)
+            resampled_chunk = resample(chunk, num_samples)
+            chunk = resampled_chunk.astype(np.float32)
+
+        scaled_chunk = chunk * 32768.0
+        clipped_chunk = np.clip(scaled_chunk, -32768, 32767)
+        int16_chunk = clipped_chunk.astype(np.int16)
+        samples = int16_chunk.tobytes()
+        self.outbuf.put_nowait(TTSResult(samples, False))
+        return self.is_closed and 0 or 1
+
+    async def write(self, text: str, split: bool, pause: float = 0.2):
+        start = time.time()
+        if split:
+            texts = re.split(splitter, text)
+        else:
+            texts = [text]
+
+        audio_duration = 0.0
+        audio_size = 0
+
+        for idx, text in enumerate(texts):
+            text = text.strip()
+            if not text:
+                continue
+            sub_start = time.time()
+
+            audio = await asyncio.to_thread(self.engine.generate,
+                                            text, self.sid, self.speed,
+                                            self.on_process)
+
+            if not audio or not audio.sample_rate or not audio.samples:
+                logger.error(f"tts: failed to generate audio for "
+                             f"'{text}' (audio={audio})")
+                continue
+
+            if split and idx < len(texts) - 1:  # add a pause between sentences
+                noise = np.zeros(int(audio.sample_rate * pause))
+                self.on_process(noise, 1.0)
+                audio.samples = np.concatenate([audio.samples, noise])
+
+            audio_duration += len(audio.samples) / audio.sample_rate
+            audio_size += len(audio.samples)
+            elapsed_seconds = time.time() - sub_start
+            logger.info(f"tts: generated audio for '{text}', "
+                        f"audio duration: {audio_duration:.2f}s, "
+                        f"elapsed: {elapsed_seconds:.2f}s")
+
+        elapsed_seconds = time.time() - start
+        logger.info(f"tts: generated audio in {elapsed_seconds:.2f}s, "
+                    f"audio duration: {audio_duration:.2f}s")
+
+        r = TTSResult(None, True)
+        r.elapsed = elapsed_seconds
+        r.audio_duration = audio_duration
+        r.progress = 1.0
+        r.finished = True
+        await self.outbuf.put(r)
+
+    async def close(self):
+        self.is_closed = True
+        self.outbuf.put_nowait(None)
+        logger.info("tts: stream closed")
+
+    async def read(self) -> TTSResult:
+        return await self.outbuf.get()
+
+    async def generate(self,  text: str) -> io.BytesIO:
+        start = time.time()
+        audio = await asyncio.to_thread(self.engine.generate,
+                                        text, self.sid, self.speed)
+        elapsed_seconds = time.time() - start
+        audio_duration = len(audio.samples) / audio.sample_rate
+
+        logger.info(f"tts: generated audio in {elapsed_seconds:.2f}s, "
+                    f"audio duration: {audio_duration:.2f}s, "
+                    f"sample rate: {audio.sample_rate}")
+
+        if self.target_sample_rate != audio.sample_rate:
+            audio.samples = resample(audio.samples,
+                                     int(len(audio.samples) * self.target_sample_rate / audio.sample_rate))
+            audio.sample_rate = self.target_sample_rate
+
+        output = io.BytesIO()
+        soundfile.write(output,
+                        audio.samples,
+                        samplerate=audio.sample_rate,
+                        subtype="PCM_16",
+                        format="WAV")
+        output.seek(0)
+        return output
+
+
+async def start_tts_stream(sid: int, sample_rate: int, speed: float, args) -> TTSStream:
+    engine, original_sample_rate = get_tts_engine(args)
+    return TTSStream(engine, sid, speed, sample_rate, original_sample_rate)