defmodule ChaiWeb.ChatLive do use ChaiWeb, :live_view @impl true def mount(_params, _session, socket) do {:ok, socket |> assign( messages: [], message: "", history: nil, reply_task: nil, transcribe_task: nil, caption_task: nil ) |> allow_upload(:audio, accept: :any, progress: &handle_progress/3, auto_upload: true) |> allow_upload(:image, accept: ~w(.jpg .jpeg .png), progress: &handle_progress/3, auto_upload: true )} end @impl true def render(assigns) do ~H"""
<.message_content message={message} />
<.icon name="hero-microphone-solid" class="w-4 h-4" />
<%= message.reaction %>
<.typing />
""" end defp typing(assigns) do ~H"""
""" end defp message_content(assigns) when assigns.message.image != nil do ~H""" """ end defp message_content(assigns) do ~H"""
<%= if label do %> <%= text %> <.icon :if={icon = label_to_icon(label)} name={icon} class="w-4 h-4" /> <% else %> <%= text %> <% end %>
""" end defp label_to_icon("LOC"), do: "hero-map-pin-solid" defp label_to_icon("PER"), do: "hero-user-solid" defp label_to_icon("ORG"), do: "hero-building-office-solid" defp label_to_icon("MISC"), do: nil defp handle_progress(:audio, entry, socket) when entry.done? do binary = consume_uploaded_entry(socket, entry, fn %{path: path} -> {:ok, File.read!(path)} end) # We always pre-process audio on the client into a single channel audio = Nx.from_binary(binary, :f32) {:noreply, request_transcription(socket, audio)} end defp handle_progress(:image, entry, socket) when entry.done? do filename = consume_uploaded_entry(socket, entry, fn %{path: path} -> filename = Path.basename(path) File.cp!(path, Chai.upload_path(filename)) {:ok, filename} end) {:noreply, request_caption(socket, filename)} end defp handle_progress(_name, _entry, socket), do: {:noreply, socket} @impl true def handle_event("send_message", %{"message" => text}, socket) do {:noreply, socket |> insert_message(text, user?: true) |> request_reply(text) |> assign(message: "")} end def handle_event("noop", %{}, socket) do # We need phx-change and phx-submit on the form for live uploads, # but we process the upload immediately in handle_progress/3 {:noreply, socket} end @impl true def handle_info({ref, {:reply, {text, history}}}, socket) when socket.assigns.reply_task.ref == ref do {:noreply, socket |> insert_message(text) |> assign(history: history, reply_task: nil)} end def handle_info({ref, {:transcription, text}}, socket) when socket.assigns.transcribe_task.ref == ref do {:noreply, socket |> insert_message(text, user?: true, transcribed?: true) |> request_reply(text) |> assign(transcribe_task: nil)} end def handle_info({ref, {:caption, filename, text}}, socket) when socket.assigns.caption_task.ref == ref do text = "look, an image of " <> text {:noreply, socket |> insert_message(text, user?: true, image: filename) |> request_reply(text) |> assign(caption_task: nil)} end def handle_info({_ref, {:reaction, message_id, reaction}}, socket) when reaction != nil do {:noreply, update_message(socket, message_id, &%{&1 | reaction: reaction})} end def handle_info({_ref, {:entities, message_id, entities}}, socket) when entities != [] do {:noreply, update_message(socket, message_id, &%{&1 | entities: entities})} end def handle_info(_message, socket), do: {:noreply, socket} defp insert_message(socket, text, opts \\ []) do message = %{ id: System.unique_integer(), text: text, user?: Keyword.get(opts, :user?, false), transcribed?: Keyword.get(opts, :transcribed?, false), image: Keyword.get(opts, :image), reaction: nil, entities: [] } socket = update(socket, :messages, &[message | &1]) socket = if message.image do socket else request_entities(socket, message.text, message.id) end if message.user? do request_reaction(socket, message.text, message.id) else socket end end defp update_message(socket, message_id, fun) do update(socket, :messages, fn messages -> Enum.map(messages, fn %{id: ^message_id} = message -> fun.(message) message -> message end) end) end defp request_reply(socket, text) do history = socket.assigns.history task = Task.async(fn -> {:reply, Chai.AI.generate_reply(text, history)} end) assign(socket, reply_task: task) end defp request_transcription(socket, audio) do task = Task.async(fn -> {:transcription, Chai.AI.transcribe(audio)} end) assign(socket, transcribe_task: task) end defp request_caption(socket, filename) do task = Task.async(fn -> image = filename |> Chai.upload_path() |> StbImage.read_file!() {:caption, filename, Chai.AI.describe_image(image)} end) assign(socket, caption_task: task) end defp request_reaction(socket, text, message_id) do Task.async(fn -> {:reaction, message_id, Chai.AI.get_reaction(text)} end) socket end defp request_entities(socket, text, message_id) do Task.async(fn -> {:entities, message_id, Chai.AI.get_entities(text)} end) socket end end