lib/hooks/useVisionAgent.tsx

import { useChat, type Message, UseChatHelpers } from 'ai/react';
import { toast } from 'react-hot-toast';
import { useEffect, useState } from 'react';
import { ChatEntity, MessageBase, SignedPayload } from '../types';
import { saveKVChatMessage } from '../kv/chat';
import { fetcher } from '../utils';
import {
  getCleanedUpMessages,
  CLEANED_SEPARATOR,
} from './useCleanedUpMessages';

const uploadBase64 = async (
  base64: string,
  messageId: string,
  chatId: string,
  index: number,
) => {
  const res = await fetch('data:image/png;base64,' + base64);
  const blob = await res.blob();
  const { signedUrl, publicUrl, fields } = await fetcher<SignedPayload>(
    '/api/sign',
    {
      method: 'POST',
      body: JSON.stringify({
        id: `${chatId}/${messageId}`,
        fileType: blob.type,
        fileName: `answer-${index}.${blob.type.split('/')[1]}`,
      }),
    },
  );
  const formData = new FormData();
  Object.entries(fields).forEach(([key, value]) => {
    formData.append(key, value as string);
  });
  formData.append('file', blob);

  const uploadResponse = await fetch(signedUrl, {
    method: 'POST',
    body: formData,
  });
  if (uploadResponse.ok) {
    return publicUrl;
  } else {
    throw new Error('Upload failed');
  }
};

const useVisionAgent = (chat: ChatEntity) => {
  const { messages: initialMessages, id, url } = chat;
  const {
    messages,
    append: appendRaw,
    reload,
    stop,
    isLoading,
    input,
    setInput,
    setMessages,
  } = useChat({
    sendExtraMessageFields: true,
    api: '/api/vision-agent',
    onResponse(response) {
      if (response.status !== 200) {
        toast.error(response.statusText);
      }
    },
    onFinish: async message => {
      const { logs = '', content, images } = getCleanedUpMessages(message);
      if (images?.length) {
        const publicUrls = await Promise.all(
          images.map((image, index) =>
            uploadBase64(image, message.id, id, index),
          ),
        );
        const newMessage = {
          ...message,
          content:
            logs +
            CLEANED_SEPARATOR +
            content +
            '\n' +
            publicUrls
              .map((url, index) => `![image-${index}](${url})`)
              .join('\n'),
        };
        saveKVChatMessage(id, newMessage);
      } else {
        saveKVChatMessage(id, {
          ...message,
          content: logs + CLEANED_SEPARATOR + content,
        });
      }
    },
    initialMessages: initialMessages,
    body: {
      url,
      id,
    },
  });

  const [loadingDots, setLoadingDots] = useState('');

  useEffect(() => {
    let loadingInterval: NodeJS.Timeout;

    if (isLoading) {
      loadingInterval = setInterval(() => {
        setLoadingDots(prevMessage => {
          switch (prevMessage) {
            case '':
              return '.';
            case '.':
              return '..';
            case '..':
              return '...';
            case '...':
              return '';
            default:
              return '';
          }
        });
      }, 500);
    }

    return () => {
      clearInterval(loadingInterval);
    };
  }, [isLoading]);

  useEffect(() => {
    if (
      !isLoading &&
      messages.length &&
      messages[messages.length - 1].role === 'user'
    ) {
      reload();
    }
  }, [isLoading, messages, reload]);

  const assistantLoadingMessage = {
    id: 'loading',
    content: loadingDots,
    role: 'assistant',
  };

  const messageWithLoading =
    isLoading &&
    messages.length &&
    messages[messages.length - 1].role !== 'assistant'
      ? [...messages, assistantLoadingMessage]
      : messages;

  const append: UseChatHelpers['append'] = async message => {
    await saveKVChatMessage(id, message as MessageBase);
    return appendRaw(message);
  };

  return {
    messages: messageWithLoading as MessageBase[],
    append,
    reload,
    stop,
    isLoading,
    input,
    setInput,
  };
};

export default useVisionAgent;