changing the URL again as the public server is being abused by some users

.env

@@ -1,3 +1,3 @@
 NEXT_PUBLIC_BASE_URL=https://jbilcke-hf-fishtank.hf.space
 # NEXT_PUBLIC_RENDERING_ENGINE_API=https://hysts-zeroscope-v2.hf.space
-RENDERING_ENGINE_API=https://jbilcke-hf-videochain-api.hf.space
+RENDERING_ENGINE_API=https://jbilcke-hf-videochain.hf.space

src/app/main.tsx

@@ -2,7 +2,8 @@
 
 import { ReactNode, useEffect, useRef, useState, useTransition } from "react"
 import { usePathname, useRouter, useSearchParams } from "next/navigation"
-
+import { DndProvider } from "react-dnd"
+import { HTML5Backend } from "react-dnd-html5-backend"
 
 import { SceneRenderer } from "@/components/renderer"
 
@@ -17,7 +18,7 @@ import { Switch } from "@/components/ui/switch"
 import { Label } from "@/components/ui/label"
 
 import { newRender, getRender } from "./render"
-import { InventoryEvent, InventoryItem, RenderedScene, SceneEvent } from "./types"
+import { InventoryEvent, InventoryItem, OnInventoryEvent, RenderedScene, SceneEvent } from "./types"
 import { Game, GameType } from "./games/types"
 import { defaultGame, games, getGame } from "./games"
 import { getBackground } from "@/app/queries/getBackground"
@@ -26,6 +27,8 @@ import { getActionnables } from "@/app/queries/getActionnables"
 import { Engine, EngineType, defaultEngine, engines, getEngine } from "./engines"
 import { normalizeActionnables } from "@/lib/normalizeActionnables"
 import { Inventory } from "@/components/inventory"
+import { store } from "./store"
+import { defaultActionnables } from "@/lib/defaultActionnables"
 
 const getInitialRenderedScene = (): RenderedScene => ({
   renderId: "",
@@ -57,6 +60,12 @@ export default function Main() {
 
   const [dialogue, setDialogue] = useState("")
 
+  /*
+  const [grabbedItem, setGrabbedItem] = useState<InventoryItem>()
+  const grabbedItemRef = useRef<InventoryItem | undefined>()
+  grabbedItemRef.current = grabbedItem
+  */
+
   const [isBusy, setBusy] = useState<boolean>(true)
   const busyRef = useRef(true)
 
@@ -88,6 +97,8 @@ export default function Main() {
       console.log("got the first version of our scene!", newRendered)
 
       // detect if type game type changed while we were busy
+      // note that currently we reload the whol page when tha happens,
+      // so this code isn't that useful
       if (game?.type !== gameRef?.current) {
         console.log("game type changed! aborting..")
         return
@@ -160,35 +171,54 @@ export default function Main() {
     checkRenderedLoop()
   }, [])
 
-  const handleClickOnActionnable = async (actionnable: string = "", userAction: string = "") => {
 
-    setBusy(busyRef.current = true)
+  const askGameMasterForEpicDialogue = async (lastEvent: string) => {
+
+    await startTransition(async () => {
+      // const game = getGame(gameRef.current)
+      let newDialogue = ""
+      try {
+        newDialogue = await getDialogue({ game, situation, lastEvent })
+      } catch (err) {
+        console.log(`failed to generate dialogue, let's try again maybe`)
+        try {
+          newDialogue = await getDialogue({ game, situation, lastEvent: `${lastEvent}.` })
+        } catch (err) {
+          console.log(`failed to generate dialogue.. again (but it's only a nice to have, so..)`)
+          setDialogue("")
+          return
+        }
+      }
+
+      // try to remove whatever hallucination might come up next
+      newDialogue = newDialogue.split("As the player")[0]
+      newDialogue = newDialogue.split("As they use")[0]
+      setDialogue(newDialogue)
+    })
+  }
 
-    // TODO: ask Llama2 what to do about it
-    // we need a frame and some actionnables,
-    // perhaps even some music or sound effects
+  const askGameMasterForEpicBackground = async (lastEvent: string = "") => {
 
     await startTransition(async () => {
 
-      const game = getGame(gameRef.current)
+      setBusy(busyRef.current = true) // this will be set to false once the scene finish loading
+
+      // const game = getGame(gameRef.current)
+
+      let newActionnables = [...defaultActionnables]
 
-      let newDialogue = ""
       try {
-        newDialogue = await getDialogue({ game, situation, userAction })
-        // console.log(`newDialogue:`, newDialogue)
-        setDialogue(newDialogue)
+        newActionnables = await getActionnables({ game, situation, lastEvent })
+        console.log(`newActionnables:`, newActionnables)
       } catch (err) {
-        console.log(`failed to generate dialogue (but it's only a nice to have, so..)`)
-        setDialogue("")
+        console.log(`failed to generate actionnables, using default value`)
       }
 
       try {
-        const newActionnables = await getActionnables({ game, situation, userAction })
-        console.log(`newActionnables:`, newActionnables)
 
         // todo rename this background/situation mess
         // it should be only one word
-        const newBackground = await getBackground({ game, situation, userAction, newActionnables })
+        const newBackground = await getBackground({ game, situation, lastEvent, newActionnables })
         console.log(`newBackground:`, newBackground)
         setSituation(newBackground)
 
@@ -197,11 +227,12 @@ export default function Main() {
 
           // todo we could also use useEffect
       } catch (err) {
-        console.error(`failed to get one of the mandatory entites: ${err}`)
+        console.error(`failed to get the scene: ${err}`)
       }
     })
   }
 
+
   const clickables = Array.from(new Set(rendered.segments.map(s => s.label)).values())
 
   // console.log("segments:", rendered.segments)
@@ -273,15 +304,28 @@ export default function Main() {
   }
 
   const handleSceneEvent = (event: SceneEvent, actionnable?: string) => {
+    // TODO use Zustand
+    const item = store.currentlyDraggedItem
+
     const actionnableName = actionnable || "nothing"
     let newEvent = null
     let newEventString = ""
     if (event === "HoveringNothing") {
-      newEvent = <>🔎 You are looking at: <span className="font-bold">Nothing</span></>
-      newEventString = `User is looking at nothing.`
+      if (item) {
+        newEvent = <>🔎 You are holding <span className="font-bold">&quot;{item.name}&quot;</span> and looking around, wondering how to use it.</>
+        newEventString = `User is holding "${item.name}" from their inventory and wonder how they can use it.`
+      } else {
+        newEvent = <>🔎 You are looking at the scene, looking for clues.</>
+        newEventString = `User is looking at the scene, looking for clues.`
+      }
     } else if (event === "HoveringActionnable") {
-      newEvent = <>🔎 You are looking at: <span className="font-bold">{actionnableName}</span></>
-      newEventString = `User is looking at "${actionnableName}"`
+      if (item) {
+        newEvent = <>🔎 You are holding <span className="font-bold">&quot;{item.name}&quot;</span>, waving it over <span className="font-bold">&quot;{actionnableName}&quot;</span></>
+        newEventString = `User is holding "${item.name}" from their inventory and wonder if they can use it on "${actionnableName}"`
+      } else {
+        newEvent = <>🔎 You are looking at <span className="font-bold">&quot;{actionnableName}&quot;</span></>
+        newEventString = `User is looking at "${actionnableName}"`
+      }
     } else if (event === "ClickOnNothing") {
       newEvent = <>🔎 There is nothing here.</>
       newEventString = `User clicked on nothing.` 
@@ -297,43 +341,39 @@ export default function Main() {
     }
 
     if (event === "ClickOnActionnable" || event === "ClickOnNothing") {
-      handleClickOnActionnable(actionnable, newEventString)
+      askGameMasterForEpicDialogue(newEventString)
+      askGameMasterForEpicBackground(newEventString)
     }
   }
 
-  const askGameMasterForSomeDialogue = async (userAction: string) => {
-    await startTransition(async () => {
-      const game = getGame(gameRef.current)
-      let newDialogue = ""
-      try {
-        newDialogue = await getDialogue({ game, situation, userAction })
-      } catch (err) {
-        console.log(`failed to generate dialoguee, let's try again maybe`)
-        try {
-          newDialogue = await getDialogue({ game, situation, userAction: `${userAction}.` })
-        } catch (err) {
-          console.log(`failed to generate dialogue.. again (but it's only a nice to have, so..)`)
-          setDialogue("")
-          return
-        }
-      }
-
-      // try to remove whatever hallucination might come up next
-      newDialogue = newDialogue.split("As the player")[0]
-      newDialogue = newDialogue.split("As they use")[0]
-      setDialogue(newDialogue)
-    })
-  }
-
-  const handleInventoryEvent = (event: InventoryEvent, item: InventoryItem, target?: InventoryItem) => {
+  const handleInventoryEvent: OnInventoryEvent = async (event, item, target) => {
     let newEvent = null
     let newEventString = ""
     if (newEvent === "Grabbing") {
       newEventString = `Player just grabbed "${item.name}".`
-      newEvent = <>You just grabbed <span className="font-bold">{item.name}</span></>
+      newEvent = <>You just grabbed <span className="font-bold">&quot;{item.name}&quot;</span></>
+    } else if (event === "DroppedOnActionnable") {
+      newEventString = [
+        `Player is trying to use`,
+        `"${item.name}"`,
+        item.description ? `(described as: "${item.description}")` : "",
+        `from their inventory on "${target?.title}"`,
+        target?.description ? `(described as: "${target?.description}")` : "",
+        `within the scene.`,
+        `What do you think should be the outcome?`
+      ].filter(i => i).join(" ")
+      newEvent = <>You try to use <span className="font-bold">&quot;{item.name}&quot;</span> on <span className="font-bold">&quot;{target?.title}&quot;</span></>
     } else if (event === "DroppedOnAnotherItem") {
-      newEventString = `Player is trying to use those object from their own inventory: "${item.name}" with "${target?.name}". What do you think could the combination of ${item.name} and ${target?.name} lead to? Invent a funny outcome!`
-      newEvent = <>You tried to combine <span className="font-bold">&quot;{item.name}&quot;</span> with <span className="font-bold">&quot;{target?.name}&quot;</span></>
+      newEventString = [
+        `Player is trying to use`,
+        `"${item.name}"`,
+        item.description ? `(described as: "${item.description}")` : "",
+        `on "${target?.name}"`,
+        target?.description ? `(described as: "${target?.description}")` : "",
+        `What do you think could the use or combination of ${item.name} and ${target?.name} lead to?`,
+        `Invent a funny outcome!`
+      ].filter(i => i).join(" ")
+      newEvent = <>You try to combine <span className="font-bold">&quot;{item.name}&quot;</span> with <span className="font-bold">&quot;{target?.title}&quot;</span></>
     } else if (event === "ClickOnItem") {
       newEventString = `Player is inspecting "${item.name}" from their inventory, which has the following description: "${item.description}". Can you invent a funny back story?`
       newEvent = <>🔎 You are inspecting <span className="font-bold">&quot;{item.name}&quot;.</span> {item.description}</>
@@ -345,11 +385,16 @@ export default function Main() {
       setLastEvent(newEvent)
     }
 
-    if (event === "DroppedOnAnotherItem" || event === "ClickOnItem") {
-      askGameMasterForSomeDialogue(newEventString)
+    if (event === "DroppedOnAnotherItem" || event === "ClickOnItem" || event === "DroppedOnActionnable") {
+      askGameMasterForEpicDialogue(newEventString)
+    }
+
+    if (event === "DroppedOnActionnable") {
+      askGameMasterForEpicBackground(newEventString)
     }
   }
 
+
   // determine when to show the spinner
   const isLoading = isBusy || rendered.status === "pending"
 
@@ -401,41 +446,44 @@ export default function Main() {
         </div>
       </div>
 
-      <div className={[
-        "flex flex-col w-full pt-4 space-y-3 text-gray-50 dark:text-gray-50",
-        getGame(gameRef.current).className // apply the game theme
-      ].join(" ")}>
-        <div className="flex flex-row">
-          <div className="text-xl px-2">{lastEvent}</div>
-        </div>
-        <Inventory game={game} onEvent={handleInventoryEvent} />
-        <div className="flex flex-row">
-          <div className="text-xl mr-2">
-            {rendered.segments.length
-              ? <span>💡 Try to click on:</span>
-              : <span>⌛ Generating clickable areas..</span>
-            }
+      <DndProvider backend={HTML5Backend}>
+        <div className={[
+          "flex flex-col w-full pt-4 space-y-3 text-gray-50 dark:text-gray-50",
+          getGame(gameRef.current).className // apply the game theme
+        ].join(" ")}
+        >
+          <div className="flex flex-row">
+            <div className="text-xl px-2">{lastEvent}</div>
+          </div>
+          <Inventory game={game} onEvent={handleInventoryEvent} />
+          <div className="flex flex-row">
+            <div className="text-xl mr-2">
+              {rendered.segments.length
+                ? <span>💡 Try to click on:</span>
+                : <span>⌛ Generating areas for clicks and drag & drop, please wait..</span>
+              }
+            </div>
+            {clickables.map((clickable, i) => 
+            <div key={i} className="flex flex-row text-xl mr-2">
+              <div className="">{clickable}</div>
+              {i < (clickables.length - 1) ? <div>,</div> : null}
+            </div>)}
           </div>
-          {clickables.map((clickable, i) => 
-          <div key={i} className="flex flex-row text-xl mr-2">
-            <div className="">{clickable}</div>
-            {i < (clickables.length - 1) ? <div>,</div> : null}
-          </div>)}
+          <SceneRenderer
+            rendered={rendered}
+            onEvent={handleSceneEvent}
+            isLoading={isLoading}
+            game={game}
+            engine={engine}
+            debug={debug}
+          />
+          <div
+            className="text-xl rounded-xl backdrop-blur-sm bg-white/10 p-4"
+            style={{
+              textShadow: "1px 0px 2px #000000ab"
+            }}>{dialogue}</div>
         </div>
-        <SceneRenderer
-          rendered={rendered}
-          onEvent={handleSceneEvent}
-          isLoading={isLoading}
-          game={game}
-          engine={engine}
-          debug={debug}
-        />
-        <div
-          className="text-xl rounded-xl backdrop-blur-sm bg-white/10 p-4"
-          style={{
-            textShadow: "1px 0px 2px #000000ab"
-          }}>{dialogue}</div>
-      </div>
+      </DndProvider>
     </div>
   )
 }

src/app/queries/getActionnables.ts

@@ -9,14 +9,14 @@ import { normalizeActionnables } from "@/lib/normalizeActionnables"
 export const getActionnables = async ({
   game,
   situation = "",
-  userAction = "",
+  lastEvent = "",
 }: {
   game: Game;
   situation: string;
-  userAction: string;
+  lastEvent: string;
 }) => {
 
-  const { currentPrompt, initialPrompt, userSituationPrompt } = getBase({ game, situation, userAction })
+  const { currentPrompt, initialPrompt, userSituationPrompt } = getBase({ game, situation, lastEvent })
 
   const basePrompt = initialPrompt !== currentPrompt
     ? `Here is some context information about the initial scene: ${initialPrompt}`

src/app/queries/getBackground.ts

@@ -7,12 +7,12 @@ import { predict } from "./predict"
 export const getBackground = async ({
   game,
   situation = "",
-  userAction = "",
+  lastEvent = "",
   newActionnables = [],
 }: {
   game: Game;
   situation: string;
-  userAction: string;
+  lastEvent: string;
   newActionnables: string[],
 }) => {
 
@@ -23,7 +23,7 @@ export const getBackground = async ({
   } = getBase({
     game,
     situation,
-    userAction
+    lastEvent
   })
 
   const basePrompt = initialPrompt !== currentPrompt
@@ -38,9 +38,9 @@ Here is the original scene in which the user was located at first, which will in
         `You are the AI game master of a role video game.`,
         basePrompt,
         `You are going to receive new information about the current whereabouts of the player.`,
-        `Please write a caption for the next plausible scene to display in intricate details: the environment, lights, era, characters, objects, textures, light etc.`,
+        `Please write a photo caption for the next plausible scene to display in intricate details: the environment, lights, era, characters, objects, textures, light etc.`,
         `You MUST include the following important objects that the user can click on: ${newActionnables}.`,
-        `Be straight to the point, and do not say things like "As the player clicks on.." or "the scene shifts to" (the best is not not mention the player at all)`
+        `As this is a caption be synthetic: describe things, but don't comment on them. Be straight to the point, and do not say things like "As the player clicks on.." or "the scene shifts to" (the best is not not mention the player at all)`
       ].filter(item => item).join("\n")
     },
     {

src/app/queries/getBase.ts

@@ -3,11 +3,11 @@ import { Game } from "@/app/games/types"
 export const getBase = ({
   game,
   situation = "",
-  userAction = "",
+  lastEvent = "",
 }: {
   game: Game;
   situation: string;
-  userAction: string;
+  lastEvent: string;
 }) => {
   const initialPrompt = [...game.getScenePrompt()].join(", ")
 
@@ -17,7 +17,7 @@ export const getBase = ({
 
   const userSituationPrompt = [
     `Player is currently in "${currentPrompt}".`,
-    userAction
+    lastEvent
   ].join(" ")
 
   return { initialPrompt, currentPrompt, userSituationPrompt }

src/app/queries/getDialogue.ts

@@ -8,17 +8,17 @@ import { predict } from "./predict"
 export const getDialogue = async ({
   game,
   situation = "",
-  userAction = "",
+  lastEvent = "",
 }: {
   game: Game;
   situation: string;
-  userAction: string;
+  lastEvent: string;
 }) => {
 
-  const { currentPrompt, initialPrompt, userSituationPrompt } = getBase({ game, situation, userAction })
+  const { currentPrompt, initialPrompt, userSituationPrompt } = getBase({ game, situation, lastEvent })
 
   console.log("DEBUG", {
-    game, situation, userAction,
+    game, situation, lastEvent,
     currentPrompt,
     initialPrompt,
     userSituationPrompt,
@@ -31,7 +31,7 @@ export const getDialogue = async ({
   */
 
   const basePrompt = initialPrompt !== currentPrompt
-    ? ` You must imagine the most plausible next dialogue line from the game master, based on where the player was located before and is now, and also what the player did before and are doing now.
+    ? `You must imagine the most plausible next dialogue line from the game master, based on where the player was located before and is now, and also what the player did before and are doing now.
 Here is the original scene in which the user was located at first, which will inform you about the general settings to follow (you must respect this): "${initialPrompt}".`
     : ""
 

src/app/queries/getSound.ts

@@ -0,0 +1,77 @@
+import { Game } from "@/app/games/types"
+import { createLlamaPrompt } from "@/lib/createLlamaPrompt"
+
+import { getBase } from "./getBase"
+import { predict } from "./predict"
+
+
+export const getSound = async ({
+  game,
+  situation = "",
+  lastEvent = "",
+}: {
+  game: Game;
+  situation: string;
+  lastEvent: string;
+}) => {
+
+  const { currentPrompt, initialPrompt, userSituationPrompt } = getBase({ game, situation, lastEvent })
+
+  console.log("DEBUG", {
+    game, situation, lastEvent,
+    currentPrompt,
+    initialPrompt,
+    userSituationPrompt,
+
+  })
+  /*
+      const basePrompt = initialPrompt !== currentPrompt
+  ? `for your information, the initial game panel and scene was: ${initialPrompt}`
+  : ""
+  */
+
+  const basePrompt = initialPrompt !== currentPrompt
+    ? `Here is the original scene in which the user was located at first, which will inform you about the general settings to follow (you must respect this): "${initialPrompt}".`
+    : ""
+
+  const prompt = createLlamaPrompt([
+    {
+      role: "system",
+      content: [
+        `You are the AI game master of a role video game.`,
+        `You are going to receive new information about the current whereabouts and action of the player.`,
+        basePrompt,
+        `You must imagine a sound effect in reaction to the player action.`,
+        `Here are some examples, but don't copy them verbatim:\n`,
+        `- "An excited crowd cheering at a sports game"\n`,
+        `- "A cat is meowing for attention"\n`,
+        `- "Birds singing sweetly in a blooming garden"\n`,
+        `- "A modern synthesizer creating futuristic soundscapes"\n`,
+        `- "The vibrant beat of Brazilian samba drums"\n`,
+        `Here are some more instructions, to enhance the Qqality of your generated audio:`,
+        `1. Try to use more adjectives to describe your sound. For example: "A man is speaking clearly and slowly in a large room" is better than "A man is speaking".\n`,
+        `2. It's better to use general terms like 'man' or 'woman' instead of specific names for individuals or abstract objects that humans may not be familiar with, such as 'mummy'.\n`
+      ].filter(item => item).join("\n")
+    },
+    {
+      role: "user",
+      content: userSituationPrompt
+    }
+  ])
+
+
+  let result = ""
+  try {
+    result = await predict(prompt)
+  } catch (err) {
+    console.log(`prediction of the dialogue failed, trying again..`)
+    try {
+      result = await predict(prompt)
+    } catch (err) {
+      console.error(`prediction of the dialogue failed again!`)
+      throw new Error(`failed to generate the dialogue ${err}`)
+    }
+  }
+
+  return result
+}

src/app/queries/getSoundTrack.ts

@@ -1,22 +0,0 @@
-import { Game } from "@/app/games/types"
-import { createLlamaPrompt } from "@/lib/createLlamaPrompt"
-
-import { getBase } from "./getBase"
-import { predict } from "./predict"
-
-export const getSoundTrack = async ({
-  game,
-  situation = "",
-  actionnable = "",
-  newDialogue = "",
-  newActionnables = [],
-}: {
-  game: Game;
-  situation: string;
-  actionnable: string;
-  newDialogue: string;
-  newActionnables: string[];
-}) => {
-
-  return ""
-}

src/app/store.ts

@@ -0,0 +1,10 @@
+"use client"
+
+import { InventoryItem } from "./types"
+
+// could also be Zustand or something
+export const store: {
+  currentlyDraggedItem?: InventoryItem
+} = {
+  currentlyDraggedItem: undefined
+}

src/app/types.ts

@@ -46,9 +46,11 @@ export type RenderedSceneStatus =
 export type SceneEvent =
   | "HoveringNothing"
   | "HoveringActionnable"
+  // | "ItemIsOverActionnable"
   | "ClickOnNothing"
   | "ClickOnActionnable"
 
+  
 export interface RenderedScene {
   renderId: string
   status: RenderedSceneStatus
@@ -69,9 +71,23 @@ export type InventoryEvent =
   | "DroppedOnAnotherItem" // the item has been dropped on another inventory item
   | "DroppedBackToInventory" // the drag & drop is cancelled, the item is back in the inventory
 
-export interface InventoryItem {
+  export interface InventoryItem {
+    name: string
+    title: string
+    caption: string
+    description: string
+  }
+
+  export interface DropZoneTarget {
+    type: "InventoryItem" | "Actionnable"
+    name: string
+    title?: string
+    caption?: string
+    description?: string
+  }
+
+export type OnInventoryEvent = (event: InventoryEvent, item: InventoryItem, target?: {
   name: string
-  title: string
-  caption: string
-  description: string
-}
+  title?: string
+  description?: string
+ }) => void

src/components/inventory/draggable-item.tsx

@@ -1,26 +1,39 @@
-
+import { useEffect, useRef } from "react"
 import Image from "next/image"
 import { useDrag, useDrop } from "react-dnd"
 
 import { Game } from "@/app/games/types"
 import { Tooltip, TooltipTrigger, TooltipContent } from "@/components/ui/tooltip"
-import { InventoryEvent, InventoryItem } from "@/app/types"
-import { useEffect } from "react"
+import { DropZoneTarget, InventoryEvent, InventoryItem, OnInventoryEvent } from "@/app/types"
 
-type DropResult = InventoryItem
+import { store } from "@/app/store"
 
 export function DraggableItem({ game, item, onEvent }: {
-  game: Game;
-  item: InventoryItem;
-  onEvent: (event: InventoryEvent, item: InventoryItem, target?: InventoryItem) => void
+  game: Game
+  item: InventoryItem
+  onEvent: OnInventoryEvent
 }) {
   const [{ isDragging }, drag] = useDrag(() => ({
     type: "item",
     item,
     end: (item, monitor) => {
-      const dropResult = monitor.getDropResult<InventoryItem>()
-      if (item && dropResult) {
-        onEvent("DroppedOnAnotherItem", item, dropResult)
+      const target = monitor.getDropResult<DropZoneTarget>()
+      if (item && target) {
+        if ( target.type === "InventoryItem" && item.name === target.name) {
+          // user is trying to drop the item on itself.. that's not possible
+          return
+        }
+        onEvent(
+          target.type === "Actionnable"
+            ? "DroppedOnActionnable"
+            : "DroppedOnAnotherItem",
+          item,
+          {
+            name: target.name,
+            title: target.title,
+            description: target.description
+          }
+        )
       }
     },
     collect: (monitor) => ({
@@ -31,7 +44,12 @@ export function DraggableItem({ game, item, onEvent }: {
 
   const [{ isOver, canDrop }, drop] = useDrop({
     accept: "item",
-    drop: () => (item),
+    drop: (): DropZoneTarget => ({
+      type: "InventoryItem",
+      name: item.name,
+      title: item.title,
+      description: item.description
+    }),
     collect: (monitor) => ({
       isOver: monitor.isOver(),
       canDrop: monitor.canDrop(),
@@ -40,8 +58,10 @@ export function DraggableItem({ game, item, onEvent }: {
 
   useEffect(() => {
     if (isDragging) {
+      store.currentlyDraggedItem = item
       onEvent("Grabbing", item)
     } else {
+      store.currentlyDraggedItem = undefined
       onEvent("DroppedBackToInventory", item)
     }
   }, [isDragging])

src/components/inventory/index.tsx

@@ -1,29 +1,24 @@
-import { DndProvider } from "react-dnd"
-import { HTML5Backend } from "react-dnd-html5-backend"
-
 import { Game } from "@/app/games/types"
 import { DraggableItem } from "./draggable-item"
-import { InventoryEvent, InventoryItem } from "@/app/types"
+import { OnInventoryEvent } from "@/app/types"
 
 export function Inventory({
   game,
   onEvent
 }: {
   game: Game;
-  onEvent: (event: InventoryEvent, item: InventoryItem, target?: InventoryItem) => void
+  onEvent: OnInventoryEvent;
 }) {  
   return (
-    <DndProvider backend={HTML5Backend}>
-      <div className="grid grid-cols-6 sm:grid-cols-8 md:grid-cols-10 lg:grid-cols-12 gap-4 w-full bg-stone-500 p-4 rounded-xl">
-        {game.inventory.map(item => (
-          <DraggableItem
-            key={item.name}
-            game={game}
-            item={item}
-            onEvent={onEvent}
-          />
-        ))}
-      </div>
-    </DndProvider>
+    <div className="grid grid-cols-6 sm:grid-cols-8 md:grid-cols-10 lg:grid-cols-12 gap-4 w-full bg-stone-500 p-4 rounded-xl">
+      {game.inventory.map(item => (
+        <DraggableItem
+          key={item.name}
+          game={game}
+          item={item}
+          onEvent={onEvent}
+        />
+      ))}
+    </div>
   )
 }

src/components/renderer/cartesian-image.tsx

@@ -1,5 +1,5 @@
-import { useRef } from "react"
-import { SceneEventHandler } from "./types"
+import { useEffect, useRef } from "react"
+import { MouseEventHandler } from "./types"
 import { RenderedScene } from "@/app/types"
 
 export function CartesianImage({
@@ -9,12 +9,53 @@ export function CartesianImage({
   debug
 }: {
   rendered: RenderedScene
-  onEvent: SceneEventHandler
+  onEvent: MouseEventHandler
   className?: string
   debug?: boolean
 }) {
 
   const ref = useRef<HTMLImageElement>(null)
+
+  const cacheRef = useRef("")
+  useEffect(() => {
+    const listener = (e: DragEvent) => {
+      if (!ref.current) { return }
+
+      // TODO: check if we are currently dragging an object
+      // if yes, then we should check if clientX and clientY are matching the 
+      const boundingRect = ref.current.getBoundingClientRect()
+
+      // abort if we are not currently dragging over our display area
+      if (e.clientX < boundingRect.left) { return }
+      if (e.clientX > (boundingRect.left + boundingRect.width)) { return }
+      if (e.clientY < boundingRect.top) { return }
+      if (e.clientY > (boundingRect.top + boundingRect.height)) { return }
+
+      const containerX = e.clientX - boundingRect.left
+      const containerY = e.clientY - boundingRect.top
+    
+      const relativeX = containerX / boundingRect.width
+      const relativeY = containerY / boundingRect.height
+
+      const key = `${relativeX},${relativeY}`
+
+      // to avoid use
+      if (cacheRef.current === key) {
+        return
+      }
+      // console.log(`DRAG: calling onEvent("hover", ${relativeX}, ${relativeY})`)
+
+      cacheRef.current = key
+      onEvent("hover", relativeX, relativeY)
+    }
+
+    document.addEventListener('drag', listener)
+
+    return () => {
+      document.removeEventListener('drag', listener)
+    }
+  }, [onEvent])
+  
   const handleEvent = async (event: React.MouseEvent<HTMLImageElement, MouseEvent>, isClick: boolean) => {
 
     if (!ref.current) {
@@ -35,6 +76,7 @@ export function CartesianImage({
     const relativeY = containerY / boundingRect.height
 
     const eventType = isClick ? "click" : "hover"
+
     onEvent(eventType, relativeX, relativeY)
   }
 

src/components/renderer/cartesian-video.tsx

@@ -1,5 +1,5 @@
-import { useRef } from "react"
-import { SceneEventHandler } from "./types"
+import { useEffect, useRef } from "react"
+import { MouseEventHandler } from "./types"
 import { RenderedScene } from "@/app/types"
 
 export function CartesianVideo({
@@ -9,11 +9,53 @@ export function CartesianVideo({
   debug,
 }: {
   rendered: RenderedScene
-  onEvent: SceneEventHandler
+  onEvent: MouseEventHandler
   className?: string
   debug?: boolean
 }) {
   const ref = useRef<HTMLVideoElement>(null)
+
+
+  const cacheRef = useRef("")
+  useEffect(() => {
+    const listener = (e: DragEvent) => {
+      if (!ref.current) { return }
+
+      // TODO: check if we are currently dragging an object
+      // if yes, then we should check if clientX and clientY are matching the 
+      const boundingRect = ref.current.getBoundingClientRect()
+
+      // abort if we are not currently dragging over our display area
+      if (e.clientX < boundingRect.left) { return }
+      if (e.clientX > (boundingRect.left + boundingRect.width)) { return }
+      if (e.clientY < boundingRect.top) { return }
+      if (e.clientY > (boundingRect.top + boundingRect.height)) { return }
+
+      const containerX = e.clientX - boundingRect.left
+      const containerY = e.clientY - boundingRect.top
+    
+      const relativeX = containerX / boundingRect.width
+      const relativeY = containerY / boundingRect.height
+
+      const key = `${relativeX},${relativeY}`
+
+      // to avoid use
+      if (cacheRef.current === key) {
+        return
+      }
+      // console.log(`DRAG: calling onEvent("hover", ${relativeX}, ${relativeY})`)
+
+      cacheRef.current = key
+      onEvent("hover", relativeX, relativeY)
+    }
+
+    document.addEventListener('drag', listener)
+
+    return () => {
+      document.removeEventListener('drag', listener)
+    }
+  }, [onEvent])
+  
   const handleEvent = (event: React.MouseEvent<HTMLVideoElement, MouseEvent>, isClick: boolean) => {
 
     if (!ref.current) {

src/components/renderer/index.tsx

@@ -1,14 +1,16 @@
 import { useEffect, useRef, useState } from "react"
 
-import { ImageSegment, RenderedScene, SceneEvent } from "@/app/types"
+import { DropZoneTarget, ImageSegment, RenderedScene, SceneEvent } from "@/app/types"
 import { ProgressBar } from "../misc/progress"
 import { Game } from "@/app/games/types"
 import { Engine } from "@/app/engines"
 import { CartesianImage } from "./cartesian-image"
-import { SceneEventHandler, SceneEventType } from "./types"
+import { MouseEventHandler, MouseEventType } from "./types"
 import { CartesianVideo } from "./cartesian-video"
 import { SphericalImage } from "./spherical-image"
 import { useImageDimension } from "@/lib/useImageDimension"
+import { useDrop } from "react-dnd"
+import { formatActionnableName } from "@/lib/formatActionnableName"
 
 export const SceneRenderer = ({
   rendered,
@@ -16,7 +18,7 @@ export const SceneRenderer = ({
   isLoading,
   game,
   engine,
-  debug
+  debug,
 }: {
   rendered: RenderedScene
   onEvent: (event: SceneEvent, actionnable?: string) => void
@@ -29,11 +31,26 @@ export const SceneRenderer = ({
   const canvasRef = useRef<HTMLCanvasElement | null>(null)
   const contextRef = useRef<CanvasRenderingContext2D | null>(null)
   const [actionnable, setActionnable] = useState<string>("")
+  const actionnableRef = useRef<string>("")
   const [progressPercent, setProcessPercent] = useState(0)
   const progressRef = useRef(0)
   const isLoadingRef = useRef(isLoading)
   const maskDimension = useImageDimension(rendered.maskUrl)
 
+  const [{ isOver, canDrop }, drop] = useDrop({
+    accept: "item",
+    drop: (): DropZoneTarget => ({
+      type: "Actionnable",
+      name: actionnable,
+      title: formatActionnableName(actionnable),
+      description: ""
+    }),
+    collect: (monitor) => ({
+      isOver: monitor.isOver(),
+      canDrop: monitor.canDrop(),
+    }),
+  })
+
   useEffect(() => {
     if (!rendered.maskUrl) {
       return
@@ -104,7 +121,7 @@ export const SceneRenderer = ({
   }
 
   // note: coordinates must be between 0 and 1
-  const handleMouseEvent: SceneEventHandler = async (type: SceneEventType, relativeX: number, relativeY: number) => {
+  const handleMouseEvent: MouseEventHandler = async (type: MouseEventType, relativeX: number, relativeY: number) => {
     if (!contextRef.current) return; // Return early if mask image has not been loaded yet
     if (!rendered.maskUrl) return;
     
@@ -134,7 +151,7 @@ export const SceneRenderer = ({
       }
 
       // update the actionnable immediately, so we can show the hand / finger cursor pointer
-      setActionnable(newSegment.label)
+      setActionnable(actionnableRef.current = newSegment.label)
     }
 
     if (type === "click") {
@@ -181,7 +198,7 @@ export const SceneRenderer = ({
   }, [isLoading, rendered.assetUrl, engine?.type])
 
   return (
-    <div className="w-full pt-2">
+    <div className="w-full pt-2" ref={drop}>
       <div
         className={[
           "relative border-2 border-gray-50 rounded-xl overflow-hidden min-h-[512px]",

src/components/renderer/spherical-image.tsx

@@ -4,7 +4,7 @@ import { LensflarePlugin, ReactPhotoSphereViewer } from "react-photo-sphere-view
 
 import { RenderedScene } from "@/app/types"
 
-import { SceneEventHandler } from "./types"
+import { MouseEventHandler } from "./types"
 import { useImageDimension } from "@/lib/useImageDimension"
 import { lightSourceNames } from "@/lib/lightSourceNames"
 
@@ -17,7 +17,7 @@ export function SphericalImage({
   debug,
 }: {
   rendered: RenderedScene
-  onEvent: SceneEventHandler
+  onEvent: MouseEventHandler
   className?: string
   debug?: boolean
 }) {
@@ -56,6 +56,46 @@ export function SphericalImage({
   }
 
 
+  const cacheRef = useRef("")
+  useEffect(() => {
+    const listener = (e: DragEvent) => {
+      if (!rootContainerRef.current) { return }
+
+      // TODO: check if we are currently dragging an object
+      // if yes, then we should check if clientX and clientY are matching the 
+      const boundingRect = rootContainerRef.current.getBoundingClientRect()
+
+      // abort if we are not currently dragging over our display area
+      if (e.clientX < boundingRect.left) { return }
+      if (e.clientX > (boundingRect.left + boundingRect.width)) { return }
+      if (e.clientY < boundingRect.top) { return }
+      if (e.clientY > (boundingRect.top + boundingRect.height)) { return }
+
+      const containerX = e.clientX - boundingRect.left
+      const containerY = e.clientY - boundingRect.top
+    
+      const relativeX = containerX / boundingRect.width
+      const relativeY = containerY / boundingRect.height
+
+      const key = `${relativeX},${relativeY}`
+
+      // to avoid use
+      if (cacheRef.current === key) {
+        return
+      }
+      // console.log(`DRAG: calling onEvent("hover", ${relativeX}, ${relativeY})`)
+
+      cacheRef.current = key
+      onEvent("hover", relativeX, relativeY)
+    }
+
+    document.addEventListener('drag', listener)
+
+    return () => {
+      document.removeEventListener('drag', listener)
+    }
+  }, [onEvent])
+
   useEffect(() => {
     const task = async () => {
       // console.log("SphericalImage: useEffect")

src/components/renderer/types.ts

@@ -1,3 +1,3 @@
-export type SceneEventType = 'hover' | 'click' | 'back'
+export type MouseEventType = "hover" | "click"
 
-export type SceneEventHandler = (type: SceneEventType, x: number, y: number) => Promise<void>
+export type MouseEventHandler = (type: MouseEventType, x: number, y: number) => Promise<void>

src/lib/defaultActionnables.ts

@@ -0,0 +1,13 @@
+export const defaultActionnables = [
+  "door",
+  "rock",
+  "window",
+  "table",
+  "ground",
+  "sky",
+  "object",
+  "tree",
+  "wall",
+  "floor"
+  // but we still only want 10 here
+  ]

src/lib/formatActionnableName.ts

@@ -0,0 +1,4 @@
+export const formatActionnableName = (input: string) => {
+  input = input.replaceAll("-", " ")
+  return input.charAt(0).toUpperCase() + input.slice(1)
+}

src/lib/normalizeActionnables.ts

@@ -1,3 +1,4 @@
+import { defaultActionnables } from "./defaultActionnables"
 import { lightSourceNames } from "./lightSourceNames"
 
 export function normalizeActionnables(rawActionnables: string[]) {
@@ -10,16 +11,7 @@ export function normalizeActionnables(rawActionnables: string[]) {
   const deduplicated = new Set<string>([
     ...tmp,
     // in case result is too small, we add a reserve of useful words here
-    "door",
-    "rock",
-    "window",
-    "table",
-    "ground",
-    "sky",
-    "object",
-    "tree",
-    "wall",
-    "floor"
+    ...defaultActionnables,
     // but we still only want 10 here
     ].slice(0, 10)
   )