import type { StartAvatarResponse } from "@heygen/streaming-avatar"; import StreamingAvatar, { AvatarQuality, StreamingEvents, TaskMode, TaskType, VoiceEmotion, } from "@heygen/streaming-avatar"; import { Button, Card, CardBody, CardFooter, Divider, Spinner, Chip, Tabs, Tab, } from "@nextui-org/react"; import { useEffect, useRef, useState } from "react"; import { useMemoizedFn, usePrevious } from "ahooks"; import InteractiveAvatarTextInput from "./InteractiveAvatarTextInput"; export default function InteractiveAvatar() { const [isLoadingSession, setIsLoadingSession] = useState(false); const [isLoadingRepeat, setIsLoadingRepeat] = useState(false); const [stream, setStream] = useState(); const [debug, setDebug] = useState(); // Hardcoded values for avatarId and knowledgeId const [knowledgeId] = useState("c60c40259b034917b235da7d007002b6"); const [avatarId] = useState("eb0a8cc8046f476da551a5559fbb5c82"); const [language] = useState("en"); const [data, setData] = useState(); const [text, setText] = useState(""); const mediaStream = useRef(null); const avatar = useRef(null); const [chatMode, setChatMode] = useState("text_mode"); const [isUserTalking, setIsUserTalking] = useState(false); async function fetchAccessToken() { try { const response = await fetch("/api/get-access-token", { method: "POST", }); const token = await response.text(); console.log("Access Token:", token); return token; } catch (error) { console.error("Error fetching access token:", error); } return ""; } async function startSession() { setIsLoadingSession(true); const newToken = await fetchAccessToken(); avatar.current = new StreamingAvatar({ token: newToken, }); avatar.current.on(StreamingEvents.AVATAR_START_TALKING, (e) => { console.log("Avatar started talking", e); }); avatar.current.on(StreamingEvents.AVATAR_STOP_TALKING, (e) => { console.log("Avatar stopped talking", e); }); avatar.current.on(StreamingEvents.STREAM_DISCONNECTED, () => { console.log("Stream disconnected"); endSession(); }); avatar.current?.on(StreamingEvents.STREAM_READY, (event) => { console.log(">>>>> Stream ready:", event.detail); setStream(event.detail); }); avatar.current?.on(StreamingEvents.USER_START, (event) => { console.log(">>>>> User started talking:", event); setIsUserTalking(true); }); avatar.current?.on(StreamingEvents.USER_STOP, (event) => { console.log(">>>>> User stopped talking:", event); setIsUserTalking(false); }); try { const res = await avatar.current.createStartAvatar({ quality: AvatarQuality.Low, avatarName: avatarId, knowledgeId: knowledgeId, voice: { rate: 1.5, // 0.5 ~ 1.5 emotion: VoiceEmotion.EXCITED, }, language: language, disableIdleTimeout: true, }); setData(res); await avatar.current?.startVoiceChat({ useSilencePrompt: false, }); setChatMode("voice_mode"); } catch (error) { console.error("Error starting avatar session:", error); } finally { setIsLoadingSession(false); } } async function handleSpeak() { setIsLoadingRepeat(true); if (!avatar.current) { setDebug("Avatar API not initialized"); return; } await avatar.current .speak({ text: text, taskType: TaskType.REPEAT, taskMode: TaskMode.SYNC }) .catch((e) => { setDebug(e.message); }); setIsLoadingRepeat(false); } async function handleInterrupt() { if (!avatar.current) { setDebug("Avatar API not initialized"); return; } await avatar.current.interrupt().catch((e) => { setDebug(e.message); }); } async function endSession() { await avatar.current?.stopAvatar(); setStream(undefined); } const handleChangeChatMode = useMemoizedFn(async (v) => { if (v === chatMode) { return; } if (v === "text_mode") { avatar.current?.closeVoiceChat(); } else { await avatar.current?.startVoiceChat(); } setChatMode(v); }); const previousText = usePrevious(text); useEffect(() => { if (!previousText && text) { avatar.current?.startListening(); } else if (previousText && !text) { avatar?.current?.stopListening(); } }, [text, previousText]); useEffect(() => { return () => { endSession(); }; }, []); useEffect(() => { if (stream && mediaStream.current) { mediaStream.current.srcObject = stream; mediaStream.current.onloadedmetadata = () => { mediaStream.current!.play(); setDebug("Playing"); }; } }, [mediaStream, stream]); return (
{stream ? (
) : !isLoadingSession ? (
) : ( )}
{ handleChangeChatMode(v); }} > {chatMode === "text_mode" ? (
{text && ( Listening )}
) : (
)}
); }