import { useChat, type Message, UseChatHelpers } from 'ai/react'; import { toast } from 'react-hot-toast'; import { useEffect, useState } from 'react'; import { ChatEntity, MessageBase, SignedPayload } from '../types'; import { saveKVChatMessage } from '../kv/chat'; import { fetcher } from '../utils'; import { getCleanedUpMessages, CLEANED_SEPARATOR, } from './useCleanedUpMessages'; const uploadBase64 = async ( base64: string, messageId: string, chatId: string, index: number, ) => { const res = await fetch('data:image/png;base64,' + base64); const blob = await res.blob(); const { signedUrl, publicUrl, fields } = await fetcher( '/api/sign', { method: 'POST', body: JSON.stringify({ id: `${chatId}/${messageId}`, fileType: blob.type, fileName: `answer-${index}.${blob.type.split('/')[1]}`, }), }, ); const formData = new FormData(); Object.entries(fields).forEach(([key, value]) => { formData.append(key, value as string); }); formData.append('file', blob); const uploadResponse = await fetch(signedUrl, { method: 'POST', body: formData, }); if (uploadResponse.ok) { return publicUrl; } else { throw new Error('Upload failed'); } }; const useVisionAgent = (chat: ChatEntity) => { const { messages: initialMessages, id, url } = chat; const { messages, append: appendRaw, reload, stop, isLoading, input, setInput, setMessages, } = useChat({ sendExtraMessageFields: true, api: '/api/vision-agent', onResponse(response) { if (response.status !== 200) { toast.error(response.statusText); } }, onFinish: async message => { const { logs = '', content, images } = getCleanedUpMessages(message); if (images?.length) { const publicUrls = await Promise.all( images.map((image, index) => uploadBase64(image, message.id, id, index), ), ); const newContent = publicUrls.reduce((accum, url, index) => { return accum.replace( `![answers-${index}](/loading.gif)`, `![answers-${index}](${url})`, ); }, content); const newMessage = { ...message, content: logs + CLEANED_SEPARATOR + newContent, }; console.log(messages); setMessages([...messages, newMessage]); saveKVChatMessage(id, newMessage); } else { saveKVChatMessage(id, { ...message, content: logs + CLEANED_SEPARATOR + content, }); } }, initialMessages: initialMessages, body: { url, id, }, }); const [loadingDots, setLoadingDots] = useState(''); useEffect(() => { let loadingInterval: NodeJS.Timeout; if (isLoading) { loadingInterval = setInterval(() => { setLoadingDots(prevMessage => { switch (prevMessage) { case '': return '.'; case '.': return '..'; case '..': return '...'; case '...': return ''; default: return ''; } }); }, 500); } return () => { clearInterval(loadingInterval); }; }, [isLoading]); useEffect(() => { if ( !isLoading && messages.length && messages[messages.length - 1].role === 'user' ) { reload(); } }, [isLoading, messages, reload]); const assistantLoadingMessage = { id: 'loading', content: loadingDots, role: 'assistant', }; const messageWithLoading = isLoading && messages.length && messages[messages.length - 1].role !== 'assistant' ? [...messages, assistantLoadingMessage] : messages; const append: UseChatHelpers['append'] = async message => { await saveKVChatMessage(id, message as MessageBase); return appendRaw(message); }; return { messages: messageWithLoading as MessageBase[], append, reload, stop, isLoading, input, setInput, }; }; export default useVisionAgent;