add LLM-generated story

src/app/interface/top-menu/index.tsx

@@ -1,5 +1,7 @@
 "use client"
 
+import { useState } from "react"
+
 import {
   Select,
   SelectContent,
@@ -12,7 +14,6 @@ import { cn } from "@/lib/utils"
 import { FontName, fontList, fonts } from "@/lib/fonts"
 import { Input } from "@/components/ui/input"
 import { defaultPreset, getPreset, presets } from "@/app/engine/presets"
-import { useState } from "react"
 import { useStore } from "@/app/store"
 
 export function TopMenu() {
@@ -119,7 +120,7 @@ export function TopMenu() {
       )}>
         <Label className="flex text-sm w-24">Font:</Label>
         <Select
-          defaultValue={fontList.includes(preset.font) ? preset.font : "cartoonist"}
+          defaultValue={fontList.includes(preset.font) ? preset.font : "actionman"}
           onValueChange={(value) => { setFont(value as FontName) }}
           // disabled={atLeastOnePanelIsBusy}
           disabled={true}

src/app/main.tsx

@@ -7,10 +7,11 @@ import { PresetName, defaultPreset, getPreset } from "@/app/engine/presets"
 
 import { cn } from "@/lib/utils"
 import { TopMenu } from "./interface/top-menu"
-import { FontName, defaultFont } from "@/lib/fonts"
+import { FontName, defaultFont, fontList, fonts } from "@/lib/fonts"
 import { getRandomLayoutName, layouts } from "./layouts"
 import { useStore } from "./store"
 import { Zoom } from "./interface/zoom"
+import { getStory } from "./queries/getStory"
 
 export default function Main() {
   const [_isPending, startTransition] = useTransition()
@@ -20,6 +21,9 @@ export default function Main() {
   const requestedFont = (searchParams.get('font') as FontName) || defaultFont
   const requestedPrompt = (searchParams.get('prompt') as string) || ""
 
+  const isGeneratingStory = useStore(state => state.isGeneratingStory)
+  const setGeneratingStory = useStore(state => state.setGeneratingStory)
+
   const font = useStore(state => state.font)
   const setFont = useStore(state => state.setFont)
 
@@ -53,22 +57,37 @@ export default function Main() {
   useEffect(() => {
     if (!prompt) { return }
 
-    const newLayout = getRandomLayoutName()
-    console.log("using layout " + newLayout)
-    setLayout(newLayout)
-
-    // TODO call the LLM here!
-    const panelPrompt = preset.imagePrompt(prompt).join(", ")
-    console.log("panelPrompt:", panelPrompt)
-
-    // what we want is for it to invent a small "story"
-    // we are going to use a LLM for this, but until then let's do this:
-    setPanels([
-      `introduction scene, ${panelPrompt}`,
-      panelPrompt,
-      panelPrompt,
-      `final scene, ${panelPrompt}`,
-    ])
+    startTransition(async () => {
+
+      setGeneratingStory(true)
+
+      const newLayout = getRandomLayoutName()
+      console.log("using layout " + newLayout)
+      setLayout(newLayout)
+
+      try {
+        const llmResponse = await getStory({ preset, prompt })
+        console.log("response:", llmResponse)
+
+        // TODO call the LLM here!
+        const panelPromptPrefix = preset.imagePrompt(prompt).join(", ")
+        console.log("panel prompt prefix:", panelPromptPrefix)
+    
+        const nbPanels = 4
+        const newPanels: string[] = []
+
+        for (let p = 0; p < nbPanels; p++) {
+          const newPanel = [panelPromptPrefix, llmResponse[p] || ""]
+          newPanels.push(newPanel.map(chunk => chunk).join(", "))
+        }
+        console.log("newPanels:", newPanels)
+        setPanels(newPanels)
+      } catch (err) {
+        console.error(err)
+      } finally {
+        setGeneratingStory(false)
+      }
+    })
   }, [prompt, preset?.label]) // important: we need to react to preset changes too
 
   const LayoutElement = (layouts as any)[layout]
@@ -78,7 +97,9 @@ export default function Main() {
       <TopMenu />
       <div className={cn(
         `flex items-start w-screen h-screen pt-[120px] px-16 md:pt-[72px] overflow-y-scroll`,
-        `transition-all duration-200 ease-in-out`
+        `transition-all duration-200 ease-in-out`,
+
+        fonts.actionman.className
       )}>
         <div className="flex flex-col items-center w-full">
           <div
@@ -105,6 +126,23 @@ export default function Main() {
         </div>
       </div>
       <Zoom />
+      <div className={cn(
+        `z-20 fixed inset-0`,
+        `flex flex-row items-center justify-center`,
+        `transition-all duration-300 ease-in-out`,
+        isGeneratingStory
+          ? `bg-zinc-100/10 backdrop-blur-md`
+          : `bg-zinc-100/0 backdrop-blur-none pointer-events-none`,
+        fonts.actionman.className
+      )}>
+        <div className={cn(
+          `text-center text-lg text-stone-600 w-[70%]`,
+          isGeneratingStory ? ``: `scale-0 opacity-0`,
+          `transition-all duration-300 ease-in-out`,
+        )}>
+          Generating your story.. (hold tight)
+        </div>
+      </div>
     </div>
   )
 }

src/app/queries/getStory.ts

@@ -0,0 +1,74 @@
+import { createLlamaPrompt } from "@/lib/createLlamaPrompt"
+
+import { predict } from "./predict"
+import { Preset } from "../engine/presets"
+
+type LLMResponse = Array<{panel: number; caption: string }>
+
+export const getStory = async ({
+  preset,
+  prompt = "",
+}: {
+  preset: Preset;
+  prompt: string;
+}): Promise<string[]> => {
+
+  const query = createLlamaPrompt([
+    {
+      role: "system",
+      content: [
+        `You are a comic book author specialized in ${preset.llmPrompt}`,
+        `Please generate detailed drawing instructions for the 4 panels of a new silent comic book page.`,
+        `Give your response as a JSON array like this: \`Array<{ panel: number; caption: string}>\`.`,
+        // `Give your response as Markdown bullet points.`,
+        `Be brief in your caption don't add your own comments. Be straight to the point, and never reply things like "Sure, I can.." etc.`
+      ].filter(item => item).join("\n")
+    },
+    {
+      role: "user",
+      content: `The story is: ${prompt}`,
+    }
+  ])
+
+
+  let result = ""
+  try {
+    result = await predict(query)
+    if (!result.trim().length) {
+      throw new Error("empty result!")
+    }
+  } catch (err) {
+    console.log(`prediction of the story failed, trying again..`)
+    try {
+      result = await predict(query+".")
+      if (!result.trim().length) {
+        throw new Error("empty result!")
+      }
+    } catch (err) {
+      console.error(`prediction of the story failed again!`)
+      throw new Error(`failed to generate the story ${err}`)
+    }
+  }
+
+  console.log("Raw response from LLM:", result)
+  let tmp = result // result.split("Caption:").pop() || result
+  tmp = tmp
+    .replaceAll("}}", "}")
+    .replaceAll("]]", "]")
+    .replaceAll(",,", ",")
+
+  try {
+    // we only keep what's after the first [
+    let jsonOrNot = `[${tmp.split("[").pop() || ""}`
+
+    // and before the first ]
+    jsonOrNot = `${jsonOrNot.split("]").shift() || ""}]`
+
+    const jsonData = JSON.parse(jsonOrNot) as LLMResponse
+    const captions = jsonData.map(item => item.caption.trim())
+    return captions.map(caption => caption.split(":").pop()?.trim() || "")
+  } catch (err) {
+    console.log(`failed to read LLM response: ${err}`)
+    return []
+  }
+}

src/app/queries/{getBackground.ts → getStyle.ts}

@@ -3,54 +3,50 @@ import { createLlamaPrompt } from "@/lib/createLlamaPrompt"
 import { predict } from "./predict"
 import { Preset } from "../engine/presets"
 
-export const getBackground = async ({
+export const getStory = async ({
   preset,
-  storyPrompt = "",
-  previousPanelPrompt = "",
-  newPanelPrompt = "",
+  prompt = "",
 }: {
   preset: Preset;
-  storyPrompt: string;
-  previousPanelPrompt: string;
-  newPanelPrompt: string;
+  prompt: string;
 }) => {
 
-  const prompt = createLlamaPrompt([
+  const query = createLlamaPrompt([
     {
       role: "system",
       content: [
-        `You are a comic book author for a ${preset.llmPrompt}`,
-        `Please write in a single sentence a photo caption for the next plausible page, using a few words for each of those categories: the environment, era, characters, objects, textures, lighting.`,
-        `Separate each of those category descriptions using a comma.`,
-        `Be brief in your caption don't add your own comments. Be straight to the point, and never reply things like "As the player approaches.." or "As the player clicks.." or "the scene shifts to.." (the best is not not mention the player at all)`
+        `You are a comic book author specialized in ${preset.llmPrompt}`,
+        `You are going to be asked to write a comic book page, your mission is to answer a JSON array containing 4 items, to describe the page (one item per panel).`,
+        `Each array item should be a comic book panel caption the describe the environment, era, characters, objects, textures, lighting.`,
+        `Be brief in your caption don't add your own comments. Be straight to the point, and never reply things like "Sure, I can.." etc.`
       ].filter(item => item).join("\n")
     },
     {
       role: "user",
-      content: storyPrompt
+      content: `The story is: ${prompt}`,
     }
   ])
 
 
   let result = ""
   try {
-    result = await predict(prompt)
+    result = await predict(query)
     if (!result.trim().length) {
       throw new Error("empty result!")
     }
   } catch (err) {
-    console.log(`prediction of the background failed, trying again..`)
+    console.log(`prediction of the story failed, trying again..`)
     try {
-      result = await predict(prompt+".")
+      result = await predict(query+".")
       if (!result.trim().length) {
         throw new Error("empty result!")
       }
     } catch (err) {
-      console.error(`prediction of the background failed again!`)
-      throw new Error(`failed to generate the background ${err}`)
+      console.error(`prediction of the story failed again!`)
+      throw new Error(`failed to generate the story ${err}`)
     }
   }
 
-  const tmp = result.split("Caption:").pop() || result
+  const tmp = result // result.split("Caption:").pop() || result
   return tmp.replaceAll("\n", ", ")
 }

src/app/queries/predict.ts

@@ -17,7 +17,7 @@ export async function predict(inputs: string) {
         do_sample: true,
 
         // hard limit for max_new_tokens is 1512
-        max_new_tokens: 200, // 1150,
+        max_new_tokens: 300, // 1150,
         return_full_text: false,
       }
     })) {

src/app/store/index.ts

@@ -14,7 +14,7 @@ export const useStore = create<{
   captions: Record<string, string>
   layout: LayoutName
   zoomLevel: number
-  isGeneratingLogic: boolean
+  isGeneratingStory: boolean
   panelGenerationStatus: Record<number, boolean>
   isGeneratingText: boolean
   atLeastOnePanelIsBusy: boolean
@@ -25,7 +25,7 @@ export const useStore = create<{
   setLayout: (layout: LayoutName) => void
   setCaption: (panelId: number, caption: string) => void
   setZoomLevel: (zoomLevel: number) => void
-  setGeneratingLogic: (isGeneratingLogic: boolean) => void
+  setGeneratingStory: (isGeneratingStory: boolean) => void
   setGeneratingImages: (panelId: number, value: boolean) => void
   setGeneratingText: (isGeneratingText: boolean) => void
 }>((set, get) => ({
@@ -36,7 +36,7 @@ export const useStore = create<{
   captions: {},
   layout: getRandomLayoutName(),
   zoomLevel: 50,
-  isGeneratingLogic: false,
+  isGeneratingStory: false,
   panelGenerationStatus: {},
   isGeneratingText: false,
   atLeastOnePanelIsBusy: false,
@@ -81,7 +81,7 @@ export const useStore = create<{
   },
   setLayout: (layout: LayoutName) => set({ layout }),
   setZoomLevel: (zoomLevel: number) =>  set({ zoomLevel }),
-  setGeneratingLogic: (isGeneratingLogic: boolean) => set({ isGeneratingLogic }),
+  setGeneratingStory: (isGeneratingStory: boolean) => set({ isGeneratingStory }),
   setGeneratingImages: (panelId: number, value: boolean) => {
 
     const panelGenerationStatus: Record<number, boolean> = {

src/components/ui/toast.tsx

@@ -0,0 +1,127 @@
+import * as React from "react"
+import * as ToastPrimitives from "@radix-ui/react-toast"
+import { cva, type VariantProps } from "class-variance-authority"
+import { X } from "lucide-react"
+
+import { cn } from "@/lib/utils"
+
+const ToastProvider = ToastPrimitives.Provider
+
+const ToastViewport = React.forwardRef<
+  React.ElementRef<typeof ToastPrimitives.Viewport>,
+  React.ComponentPropsWithoutRef<typeof ToastPrimitives.Viewport>
+>(({ className, ...props }, ref) => (
+  <ToastPrimitives.Viewport
+    ref={ref}
+    className={cn(
+      "fixed top-0 z-[100] flex max-h-screen w-full flex-col-reverse p-4 sm:bottom-0 sm:right-0 sm:top-auto sm:flex-col md:max-w-[420px]",
+      className
+    )}
+    {...props}
+  />
+))
+ToastViewport.displayName = ToastPrimitives.Viewport.displayName
+
+const toastVariants = cva(
+  "group pointer-events-auto relative flex w-full items-center justify-between space-x-4 overflow-hidden rounded-md border border-stone-200 p-6 pr-8 shadow-lg transition-all data-[swipe=cancel]:translate-x-0 data-[swipe=end]:translate-x-[var(--radix-toast-swipe-end-x)] data-[swipe=move]:translate-x-[var(--radix-toast-swipe-move-x)] data-[swipe=move]:transition-none data-[state=open]:animate-in data-[state=closed]:animate-out data-[swipe=end]:animate-out data-[state=closed]:fade-out-80 data-[state=closed]:slide-out-to-right-full data-[state=open]:slide-in-from-top-full data-[state=open]:sm:slide-in-from-bottom-full dark:border-stone-800",
+  {
+    variants: {
+      variant: {
+        default: "border bg-white text-stone-950 dark:bg-stone-950 dark:text-stone-50",
+        destructive:
+          "destructive group border-red-500 bg-red-500 text-stone-50 dark:border-red-900 dark:bg-red-900 dark:text-stone-50",
+      },
+    },
+    defaultVariants: {
+      variant: "default",
+    },
+  }
+)
+
+const Toast = React.forwardRef<
+  React.ElementRef<typeof ToastPrimitives.Root>,
+  React.ComponentPropsWithoutRef<typeof ToastPrimitives.Root> &
+    VariantProps<typeof toastVariants>
+>(({ className, variant, ...props }, ref) => {
+  return (
+    <ToastPrimitives.Root
+      ref={ref}
+      className={cn(toastVariants({ variant }), className)}
+      {...props}
+    />
+  )
+})
+Toast.displayName = ToastPrimitives.Root.displayName
+
+const ToastAction = React.forwardRef<
+  React.ElementRef<typeof ToastPrimitives.Action>,
+  React.ComponentPropsWithoutRef<typeof ToastPrimitives.Action>
+>(({ className, ...props }, ref) => (
+  <ToastPrimitives.Action
+    ref={ref}
+    className={cn(
+      "inline-flex h-8 shrink-0 items-center justify-center rounded-md border border-stone-200 bg-transparent px-3 text-sm font-medium ring-offset-white transition-colors hover:bg-stone-100 focus:outline-none focus:ring-2 focus:ring-stone-950 focus:ring-offset-2 disabled:pointer-events-none disabled:opacity-50 group-[.destructive]:border-stone-100/40 group-[.destructive]:hover:border-red-500/30 group-[.destructive]:hover:bg-red-500 group-[.destructive]:hover:text-stone-50 group-[.destructive]:focus:ring-red-500 dark:border-stone-800 dark:ring-offset-stone-950 dark:hover:bg-stone-800 dark:focus:ring-stone-300 dark:group-[.destructive]:border-stone-800/40 dark:group-[.destructive]:hover:border-red-900/30 dark:group-[.destructive]:hover:bg-red-900 dark:group-[.destructive]:hover:text-stone-50 dark:group-[.destructive]:focus:ring-red-900",
+      className
+    )}
+    {...props}
+  />
+))
+ToastAction.displayName = ToastPrimitives.Action.displayName
+
+const ToastClose = React.forwardRef<
+  React.ElementRef<typeof ToastPrimitives.Close>,
+  React.ComponentPropsWithoutRef<typeof ToastPrimitives.Close>
+>(({ className, ...props }, ref) => (
+  <ToastPrimitives.Close
+    ref={ref}
+    className={cn(
+      "absolute right-2 top-2 rounded-md p-1 text-stone-950/50 opacity-0 transition-opacity hover:text-stone-950 focus:opacity-100 focus:outline-none focus:ring-2 group-hover:opacity-100 group-[.destructive]:text-red-300 group-[.destructive]:hover:text-red-50 group-[.destructive]:focus:ring-red-400 group-[.destructive]:focus:ring-offset-red-600 dark:text-stone-50/50 dark:hover:text-stone-50",
+      className
+    )}
+    toast-close=""
+    {...props}
+  >
+    <X className="h-4 w-4" />
+  </ToastPrimitives.Close>
+))
+ToastClose.displayName = ToastPrimitives.Close.displayName
+
+const ToastTitle = React.forwardRef<
+  React.ElementRef<typeof ToastPrimitives.Title>,
+  React.ComponentPropsWithoutRef<typeof ToastPrimitives.Title>
+>(({ className, ...props }, ref) => (
+  <ToastPrimitives.Title
+    ref={ref}
+    className={cn("text-sm font-semibold", className)}
+    {...props}
+  />
+))
+ToastTitle.displayName = ToastPrimitives.Title.displayName
+
+const ToastDescription = React.forwardRef<
+  React.ElementRef<typeof ToastPrimitives.Description>,
+  React.ComponentPropsWithoutRef<typeof ToastPrimitives.Description>
+>(({ className, ...props }, ref) => (
+  <ToastPrimitives.Description
+    ref={ref}
+    className={cn("text-sm opacity-90", className)}
+    {...props}
+  />
+))
+ToastDescription.displayName = ToastPrimitives.Description.displayName
+
+type ToastProps = React.ComponentPropsWithoutRef<typeof Toast>
+
+type ToastActionElement = React.ReactElement<typeof ToastAction>
+
+export {
+  type ToastProps,
+  type ToastActionElement,
+  ToastProvider,
+  ToastViewport,
+  Toast,
+  ToastTitle,
+  ToastDescription,
+  ToastClose,
+  ToastAction,
+}

src/components/ui/toaster.tsx

@@ -0,0 +1,35 @@
+"use client"
+
+import {
+  Toast,
+  ToastClose,
+  ToastDescription,
+  ToastProvider,
+  ToastTitle,
+  ToastViewport,
+} from "@/components/ui/toast"
+import { useToast } from "@/components/ui/use-toast"
+
+export function Toaster() {
+  const { toasts } = useToast()
+
+  return (
+    <ToastProvider>
+      {toasts.map(function ({ id, title, description, action, ...props }) {
+        return (
+          <Toast key={id} {...props}>
+            <div className="grid gap-1">
+              {title && <ToastTitle>{title}</ToastTitle>}
+              {description && (
+                <ToastDescription>{description}</ToastDescription>
+              )}
+            </div>
+            {action}
+            <ToastClose />
+          </Toast>
+        )
+      })}
+      <ToastViewport />
+    </ToastProvider>
+  )
+}

src/components/ui/use-toast.ts

@@ -0,0 +1,192 @@
+// Inspired by react-hot-toast library
+import * as React from "react"
+
+import type {
+  ToastActionElement,
+  ToastProps,
+} from "@/components/ui/toast"
+
+const TOAST_LIMIT = 1
+const TOAST_REMOVE_DELAY = 1000000
+
+type ToasterToast = ToastProps & {
+  id: string
+  title?: React.ReactNode
+  description?: React.ReactNode
+  action?: ToastActionElement
+}
+
+const actionTypes = {
+  ADD_TOAST: "ADD_TOAST",
+  UPDATE_TOAST: "UPDATE_TOAST",
+  DISMISS_TOAST: "DISMISS_TOAST",
+  REMOVE_TOAST: "REMOVE_TOAST",
+} as const
+
+let count = 0
+
+function genId() {
+  count = (count + 1) % Number.MAX_VALUE
+  return count.toString()
+}
+
+type ActionType = typeof actionTypes
+
+type Action =
+  | {
+      type: ActionType["ADD_TOAST"]
+      toast: ToasterToast
+    }
+  | {
+      type: ActionType["UPDATE_TOAST"]
+      toast: Partial<ToasterToast>
+    }
+  | {
+      type: ActionType["DISMISS_TOAST"]
+      toastId?: ToasterToast["id"]
+    }
+  | {
+      type: ActionType["REMOVE_TOAST"]
+      toastId?: ToasterToast["id"]
+    }
+
+interface State {
+  toasts: ToasterToast[]
+}
+
+const toastTimeouts = new Map<string, ReturnType<typeof setTimeout>>()
+
+const addToRemoveQueue = (toastId: string) => {
+  if (toastTimeouts.has(toastId)) {
+    return
+  }
+
+  const timeout = setTimeout(() => {
+    toastTimeouts.delete(toastId)
+    dispatch({
+      type: "REMOVE_TOAST",
+      toastId: toastId,
+    })
+  }, TOAST_REMOVE_DELAY)
+
+  toastTimeouts.set(toastId, timeout)
+}
+
+export const reducer = (state: State, action: Action): State => {
+  switch (action.type) {
+    case "ADD_TOAST":
+      return {
+        ...state,
+        toasts: [action.toast, ...state.toasts].slice(0, TOAST_LIMIT),
+      }
+
+    case "UPDATE_TOAST":
+      return {
+        ...state,
+        toasts: state.toasts.map((t) =>
+          t.id === action.toast.id ? { ...t, ...action.toast } : t
+        ),
+      }
+
+    case "DISMISS_TOAST": {
+      const { toastId } = action
+
+      // ! Side effects ! - This could be extracted into a dismissToast() action,
+      // but I'll keep it here for simplicity
+      if (toastId) {
+        addToRemoveQueue(toastId)
+      } else {
+        state.toasts.forEach((toast) => {
+          addToRemoveQueue(toast.id)
+        })
+      }
+
+      return {
+        ...state,
+        toasts: state.toasts.map((t) =>
+          t.id === toastId || toastId === undefined
+            ? {
+                ...t,
+                open: false,
+              }
+            : t
+        ),
+      }
+    }
+    case "REMOVE_TOAST":
+      if (action.toastId === undefined) {
+        return {
+          ...state,
+          toasts: [],
+        }
+      }
+      return {
+        ...state,
+        toasts: state.toasts.filter((t) => t.id !== action.toastId),
+      }
+  }
+}
+
+const listeners: Array<(state: State) => void> = []
+
+let memoryState: State = { toasts: [] }
+
+function dispatch(action: Action) {
+  memoryState = reducer(memoryState, action)
+  listeners.forEach((listener) => {
+    listener(memoryState)
+  })
+}
+
+type Toast = Omit<ToasterToast, "id">
+
+function toast({ ...props }: Toast) {
+  const id = genId()
+
+  const update = (props: ToasterToast) =>
+    dispatch({
+      type: "UPDATE_TOAST",
+      toast: { ...props, id },
+    })
+  const dismiss = () => dispatch({ type: "DISMISS_TOAST", toastId: id })
+
+  dispatch({
+    type: "ADD_TOAST",
+    toast: {
+      ...props,
+      id,
+      open: true,
+      onOpenChange: (open) => {
+        if (!open) dismiss()
+      },
+    },
+  })
+
+  return {
+    id: id,
+    dismiss,
+    update,
+  }
+}
+
+function useToast() {
+  const [state, setState] = React.useState<State>(memoryState)
+
+  React.useEffect(() => {
+    listeners.push(setState)
+    return () => {
+      const index = listeners.indexOf(setState)
+      if (index > -1) {
+        listeners.splice(index, 1)
+      }
+    }
+  }, [state])
+
+  return {
+    ...state,
+    toast,
+    dismiss: (toastId?: string) => dispatch({ type: "DISMISS_TOAST", toastId }),
+  }
+}
+
+export { useToast, toast }

src/lib/fonts.ts

@@ -78,16 +78,16 @@ export const digitalstrip = localFont({
 export const fonts = {
   indieflower,
   thegirlnextdoor,
-  komika,
+  // komika,
   actionman,
   karantula,
   manoskope,
-  paeteround,
-  qarmic,
-  archrival,
-  cartoonist,
-  toontime,
-  vtc,
+  // paeteround,
+  // qarmic,
+ //  archrival,
+  // cartoonist,
+  // toontime,
+  // vtc,
   digitalstrip
 }