up

package-lock.json

@@ -44,7 +44,9 @@
         "next": "13.4.10",
         "pick": "^0.0.1",
         "postcss": "8.4.26",
+        "qs": "^6.11.2",
         "react": "18.2.0",
+        "react-circular-progressbar": "^2.1.0",
         "react-day-picker": "^8.8.0",
         "react-dom": "18.2.0",
         "tailwind-merge": "^1.13.2",
@@ -56,6 +58,9 @@
         "uuid": "^9.0.0",
         "zod": "^3.21.4",
         "zustand": "^4.3.9"
+      },
+      "devDependencies": {
+        "@types/qs": "^6.9.7"
       }
     },
     "node_modules/@aashutoshrathi/word-wrap": {
@@ -1494,6 +1499,12 @@
       "resolved": "https://registry.npmjs.org/@types/prop-types/-/prop-types-15.7.5.tgz",
       "integrity": "sha512-JCB8C6SnDoQf0cNycqd/35A7MjcnK+ZTqE7judS6o7utxUCg6imJg3QK2qzHKszlTjcj2cn+NwMB2i96ubpj7w=="
     },
+    "node_modules/@types/qs": {
+      "version": "6.9.7",
+      "resolved": "https://registry.npmjs.org/@types/qs/-/qs-6.9.7.tgz",
+      "integrity": "sha512-FGa1F62FT09qcrueBA6qYTrJPVDzah9a+493+o2PCXsesWHIn27G98TsSMs3WPNbZIEj4+VJf6saSFpvD+3Zsw==",
+      "dev": true
+    },
     "node_modules/@types/react": {
       "version": "18.2.15",
       "resolved": "https://registry.npmjs.org/@types/react/-/react-18.2.15.tgz",
@@ -4824,6 +4835,20 @@
         "node": ">=6"
       }
     },
+    "node_modules/qs": {
+      "version": "6.11.2",
+      "resolved": "https://registry.npmjs.org/qs/-/qs-6.11.2.tgz",
+      "integrity": "sha512-tDNIz22aBzCDxLtVH++VnTfzxlfeK5CbqohpSqpJgj1Wg/cQbStNAz3NuqCs5vV+pjBsK4x4pN9HlVh7rcYRiA==",
+      "dependencies": {
+        "side-channel": "^1.0.4"
+      },
+      "engines": {
+        "node": ">=0.6"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
+      }
+    },
     "node_modules/queue-microtask": {
       "version": "1.2.3",
       "resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz",
@@ -4854,6 +4879,14 @@
         "node": ">=0.10.0"
       }
     },
+    "node_modules/react-circular-progressbar": {
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/react-circular-progressbar/-/react-circular-progressbar-2.1.0.tgz",
+      "integrity": "sha512-xp4THTrod4aLpGy68FX/k1Q3nzrfHUjUe5v6FsdwXBl3YVMwgeXYQKDrku7n/D6qsJA9CuunarAboC2xCiKs1g==",
+      "peerDependencies": {
+        "react": "^0.14.0 || ^15.0.0 || ^16.0.0 || ^17.0.0 || ^18.0.0"
+      }
+    },
     "node_modules/react-day-picker": {
       "version": "8.8.0",
       "resolved": "https://registry.npmjs.org/react-day-picker/-/react-day-picker-8.8.0.tgz",

package.json

@@ -45,7 +45,9 @@
     "next": "13.4.10",
     "pick": "^0.0.1",
     "postcss": "8.4.26",
+    "qs": "^6.11.2",
     "react": "18.2.0",
+    "react-circular-progressbar": "^2.1.0",
     "react-day-picker": "^8.8.0",
     "react-dom": "18.2.0",
     "tailwind-merge": "^1.13.2",
@@ -57,5 +59,8 @@
     "uuid": "^9.0.0",
     "zod": "^3.21.4",
     "zustand": "^4.3.9"
+  },
+  "devDependencies": {
+    "@types/qs": "^6.9.7"
   }
 }

src/app/main.tsx

@@ -1,6 +1,7 @@
 "use client"
 
 import { useEffect, useRef, useState, useTransition } from "react"
+import qs from "qs"
 
 import { ImageRenderer } from "@/components/business/image-renderer"
 
@@ -15,10 +16,11 @@ import {
 import { render } from "./render"
 
 import { RenderedScene } from "./types"
-import { predict } from "./predict"
 import { GameType } from "./games/types"
 import { defaultGame, games, getGame } from "./games"
-import { getPrompts } from "./prompts"
+import { getBackground } from "@/app/queries/getBackground"
+import { getDialogue } from "@/app/queries/getDialogue"
+import { getActionnables } from "@/app/queries/getActionnables"
 
 export default function Main() {
   const [isPending, startTransition] = useTransition()
@@ -28,7 +30,9 @@ export default function Main() {
     maskBase64: "",
     segments:[]
   })
-  const ref = useRef<GameType>(defaultGame)
+  const urlParams = qs.parse(window.location.search.slice(1))
+  console.log("urlParams:", urlParams)
+  const ref = useRef<GameType>(`${urlParams?.game as any}` as GameType)
   const [situation, setSituation] = useState("")
   const [scene, setScene] = useState("")
   const [dialogue, setDialogue] = useState("")
@@ -92,75 +96,31 @@ export default function Main() {
 
       const game = getGame(ref.current)
 
-      const prompts = getPrompts(game, situation, actionnable)
-
-      console.log("prompts:", prompts)
-
+      let newDialogue = ""
       try {
+        newDialogue = await getDialogue({ game, situation, actionnable })
+        console.log(`newDialogue:`, newDialogue)
+        setDialogue(newDialogue)
+      } catch (err) {
+        console.log(`failed to generate dialogue (but it's only a nice to have, so..)`)
+        setDialogue("")
+      }
 
-        console.log("ask the LLM to invent next steps..")
-
-        const rawSituation = await predict(prompts.situationPrompt)
-
-        console.log(`rawSituation: `, rawSituation)
-
-        if (!rawSituation) {
-          throw new Error("failed to generate the situation")
-        }
-        const newSituation = `${rawSituation || ""}`
-        if (!newSituation) {
-          throw new Error("failed to parse the situation")
-        }
-
-        console.log(`newSituation: `, newSituation)
-
-        const rawActionnables = await predict(prompts.actionnablesPrompt)
-        console.log(`rawActionnables: `, rawActionnables)
-
-        if (!rawActionnables) {
-          throw new Error("failed to generate the actionnables")
-        }
-
-        let newActionnables = []
-        try {
-          // we remove all [ or ]
-          const sanitized = rawActionnables.replaceAll("[", "").replaceAll("]", "")
-          newActionnables = (JSON.parse(`[${sanitized}]`) as string[]).map(item =>
-            // clean the words to remove any punctuation
-            item.replace(/\W/g, '').trim()
-          )
-
-          if (!newActionnables.length) {
-            throw new Error("no actionnables")
-          }
-        } catch (err) {
-          throw new Error("failed to parse the actionnables")
-        }
- 
-        console.log(`newActionnables: `, newActionnables)
-
-        const rawDialogue = await predict(prompts.dialoguePrompt)
-        console.log(`rawDialogue: `, rawDialogue)
-
-        if (!rawDialogue) {
-          throw new Error("failed to generate the dialogue")
-        }
-        const newDialogue = `${rawDialogue || ""}`
-        if (!newDialogue) {
-            throw new Error("failed to parse the dialogue")
-        }
-        console.log(`newDialogue: `, newDialogue)
-
+      try {
+        const newActionnables = await getActionnables({ game, situation, actionnable, newDialogue })
+        console.log(`newActionnables:`, newActionnables)
 
-        setDialogue(newDialogue)
-        setSituation(newSituation)
+        const newBackground = await getBackground({ game, situation, actionnable, newDialogue, newActionnables })
+        console.log(`newBackground:`, newBackground)
+        setSituation(newBackground)
 
         console.log("loading next scene..")
-        await loadNextScene(newSituation, newActionnables)
+        await loadNextScene(newBackground, newActionnables)
 
-        // todo we could also use useEffect
+          // todo we could also use useEffect
       } catch (err) {
-       console.error(err)
+        console.error(`failed to get one of the mandatory entites: ${err}`)
+        setLoading(false)
       }
     })
   }

src/app/prompts.ts

@@ -1,86 +0,0 @@
-import { Game } from "./games/types"
-import { createLlamaPrompt } from "@/lib/createLlamaPrompt"
-
-export const getPrompts = (game: Game, situation: string = "", actionnable: string = "") => {
-
-  const initialPrompt = [...game.getScenePrompt()].join(", ")
-
-  const currentPrompt = situation
-    ? [...game.getScenePrompt(situation)].join(", ")
-    : initialPrompt
-  
-  const userSituationPrompt = `Player is currently in "${currentPrompt}". Player clicked on the "${actionnable}".`
-
-  const baseSituationPromptIfWeHaveHistory = initialPrompt !== currentPrompt
-    ? ` You must imagine the most plausible next scene, based on where the player was located before and is now, and also what the player did before and are doing now.
-Here is the original scene in which the user was located at first, which will inform you about the general settings to follow (you must respect this): "${initialPrompt}".`
-    : ""
-
-  const situationPrompt = createLlamaPrompt([
-    {
-      role: "system",
-      content: [
-        `You are the AI game master of a role video game.`,
-        baseSituationPromptIfWeHaveHistory,
-        `You are going to receive new information about the current whereabouts of the player.`,
-        `Please write a caption for the next plausible scene to display in intricate details: the environment, lights, era, characters, objects, textures, light etc. You must include important objects, that the user can click on (eg. characters, doors, vehicles, useful objects).`
-      ].filter(item => item).join("\n")
-    },
-    {
-      role: "user",
-      content: userSituationPrompt
-    }
-  ])
-
-  const actionnablesPrompt = createLlamaPrompt([
-    {
-      role: "system",
-      content: [
-        `You are an API endpoint that can return a list of objects thare are in a scene.`,
-        `You must list basic name of things (eg. "parrot", "chest", "spaceship", "glass", "door", "person", "window", "light", "knob", "button" etc)`,
-        `The answer must be a JSON array, ie. a list of quoted strings.`
-      ].filter(item => item).join("\n")
-    },
-    {
-      role: "user",
-      content: userSituationPrompt
-    }
-  ])
-
-  const baseDialoguePromptIfWeHaveHistory = initialPrompt !== currentPrompt
-  ? `for your information, the initial game panel and scene was: "${initialPrompt}".`
-  : ""
-
-  const dialoguePrompt = createLlamaPrompt([
-    {
-      role: "system",
-      content: [
-        `You are the AI game master of a role video game.`,
-        `You are going to receive new information about the current whereabouts and action of the player.`,
-        baseDialoguePromptIfWeHaveHistory,
-        `You must imagine a funny response to speak in reaction to what the player did, like in some old point and click video games.`,
-        `Please limit yourself to only a 1 or 2 sentences, please.`,
-      ].filter(item => item).join("\n")
-    },
-    {
-      role: "user",
-      content: userSituationPrompt
-    }
-  ])
-
-
-  const prompts = {
-    initialPrompt,
-    currentPrompt,
-    userSituationPrompt,
-    baseSituationPromptIfWeHaveHistory,
-    situationPrompt,
-    actionnablesPrompt,
-    baseDialoguePromptIfWeHaveHistory,
-    dialoguePrompt,
-  }
-
-  return prompts
-}
-
-

src/app/queries/getActionnables.ts

@@ -0,0 +1,86 @@
+import { Game } from "@/app/games/types"
+import { createLlamaPrompt } from "@/lib/createLlamaPrompt"
+import { parseJsonList } from "@/lib/parseJsonList"
+
+import { getBase } from "./getBase"
+import { predict } from "./predict"
+
+export const getActionnables = async ({
+  game,
+  situation = "",
+  actionnable = "",
+  newDialogue = "",
+  // newActionnables = [],
+}: {
+  game: Game;
+  situation: string;
+  actionnable: string;
+  newDialogue: string;
+  // newActionnables: string[];
+}) => {
+
+  const { currentPrompt, initialPrompt, userSituationPrompt } = getBase(game, situation, actionnable)
+
+  const basePrompt = initialPrompt !== currentPrompt
+    ? `Here is some context information about the initial scene: ${initialPrompt}`
+    : ""
+
+  const prompt = createLlamaPrompt([
+    {
+      role: "system",
+      content: [
+        `You are an API endpoint that can return a list of objects visible in the background image of a role playing game.`,
+        basePrompt,
+        `You must list basic name of characters or visible objects (eg. "parrot", "chest", "spaceship", "glass", "door", "person", "window", "light", "knob", "button" etc) but don't list any word from abstract concepts (ig. don't say things like "secret", "danger", "next move" etc)`,
+        `The answer must be a JSON array, ie. a list of quoted strings.`
+      ].filter(item => item).join("\n")
+    },
+    {
+      role: "user",
+      content: userSituationPrompt
+    }
+  ])
+
+  let rawStringOutput = ""
+  
+  try {
+    rawStringOutput = await predict(prompt)
+  } catch (err) {
+    console.log(`prediction of the actionnables failed, trying again..`)
+    try {
+      rawStringOutput = await predict(prompt)
+    } catch (err) {
+      console.error(`prediction of the actionnables failed again!`)
+      throw new Error(`failed to generate the actionnables ${err}`)
+    }
+  }
+  
+  let result = []
+
+  try {
+    result = parseJsonList(rawStringOutput)
+
+    if (!result.length) {
+      throw new Error("no actionnables")
+    }
+  } catch (err) {
+    console.log("failed to find a valid JSON! attempting method 2..")
+
+    try {
+      const sanitized = rawStringOutput.replaceAll("[", "").replaceAll("]", "")
+      result = (JSON.parse(`[${sanitized}]`) as string[]).map(item =>
+        // clean the words to remove any punctuation
+        item.replace(/\W/g, '').trim()
+      )
+
+      if (!result.length) {
+        throw new Error("no actionnables")
+      }
+    } catch (err) {
+      console.log("failed to repair and recover a valid JSON!")
+      throw new Error("failed to parse the actionnables")
+    }
+  }
+
+  return  result
+}

src/app/queries/getBackground.ts

@@ -0,0 +1,61 @@
+import { Game } from "@/app/games/types"
+import { createLlamaPrompt } from "@/lib/createLlamaPrompt"
+
+import { getBase } from "./getBase"
+import { predict } from "./predict"
+
+export const getBackground = async ({
+  game,
+  situation = "",
+  actionnable = "",
+  newDialogue = "",
+  newActionnables = [],
+}: {
+  game: Game;
+  situation: string;
+  actionnable: string;
+  newDialogue: string;
+  newActionnables: string[];
+}) => {
+
+  const { currentPrompt, initialPrompt, userSituationPrompt } = getBase(game, situation, actionnable)
+
+  const basePrompt = initialPrompt !== currentPrompt
+    ? `You must imagine the most plausible next scene, based on where the player was located before and is now, and also what the player did before and are doing now.
+Here is the original scene in which the user was located at first, which will inform you about the general settings to follow (you must respect this): "${initialPrompt}".`
+    : ""
+
+  const prompt = createLlamaPrompt([
+    {
+      role: "system",
+      content: [
+        `You are the AI game master of a role video game.`,
+        basePrompt,
+        `You are going to receive new information about the current whereabouts of the player.`,
+        `Please write a caption for the next plausible scene to display in intricate details: the environment, lights, era, characters, objects, textures, light etc.`,
+        `You must include important objects, that the user can click on (eg. characters, doors, vehicles, useful objects).`,
+        `Be straight to the point, and do not say things like "As the player clicks on.." or "the scene shifts to" (the best is not not mention the player at all)`
+      ].filter(item => item).join("\n")
+    },
+    {
+      role: "user",
+      content: userSituationPrompt
+    }
+  ])
+
+
+  let result = ""
+  try {
+    result = await predict(prompt)
+  } catch (err) {
+    console.log(`prediction of the background failed, trying again..`)
+    try {
+      result = await predict(prompt)
+    } catch (err) {
+      console.error(`prediction of the background failed again!`)
+      throw new Error(`failed to generate the background ${err}`)
+    }
+  }
+
+  return result
+}

src/app/queries/getBase.ts

@@ -0,0 +1,13 @@
+import { Game } from "@/app/games/types"
+
+export const getBase = (game: Game, situation: string = "", actionnable: string = "") => {
+  const initialPrompt = [...game.getScenePrompt()].join(", ")
+
+  const currentPrompt = situation
+    ? [...game.getScenePrompt(situation)].join(", ")
+    : initialPrompt
+
+  const userSituationPrompt = `Player is currently in "${currentPrompt}". Player clicked on the "${actionnable}".`
+
+  return { initialPrompt, currentPrompt, userSituationPrompt }
+}

src/app/queries/getDialogue.ts

@@ -0,0 +1,67 @@
+import { Game } from "@/app/games/types"
+import { createLlamaPrompt } from "@/lib/createLlamaPrompt"
+
+import { getBase } from "./getBase"
+import { predict } from "./predict"
+
+
+export const getDialogue = async ({
+  game,
+  situation = "",
+  actionnable = "",
+  // newDialogue = "",
+  // newActionnables = [],
+}: {
+  game: Game;
+  situation: string;
+  actionnable: string;
+  // newDialogue: string;
+  // newActionnables: string[];
+}) => {
+
+  const { currentPrompt, initialPrompt, userSituationPrompt } = getBase(game, situation, actionnable)
+
+  /*
+      const basePrompt = initialPrompt !== currentPrompt
+  ? `for your information, the initial game panel and scene was: ${initialPrompt}`
+  : ""
+  */
+
+  const basePrompt = initialPrompt !== currentPrompt
+    ? ` You must imagine the most plausible next dialogue line from the game master, based on where the player was located before and is now, and also what the player did before and are doing now.
+Here is the original scene in which the user was located at first, which will inform you about the general settings to follow (you must respect this): "${initialPrompt}".`
+    : ""
+
+  const prompt = createLlamaPrompt([
+    {
+      role: "system",
+      content: [
+        `You are the AI game master of a role video game.`,
+        `You are going to receive new information about the current whereabouts and action of the player.`,
+        basePrompt,
+        `You must imagine a funny response to speak in reaction to what the player did, like in some old point and click video games.`,
+        `Please limit yourself to only a 1 or 2 sentences, please.`,
+      ].filter(item => item).join("\n")
+    },
+    {
+      role: "user",
+      content: userSituationPrompt
+    }
+  ])
+
+
+  let result = ""
+  try {
+    result = await predict(prompt)
+  } catch (err) {
+    console.log(`prediction of the dialogue failed, trying again..`)
+    try {
+      result = await predict(prompt)
+    } catch (err) {
+      console.error(`prediction of the dialogue failed again!`)
+      throw new Error(`failed to generate the dialogue ${err}`)
+    }
+  }
+
+  return result
+}

src/app/queries/getMusicTrack.ts

@@ -0,0 +1,21 @@
+import { Game } from "@/app/games/types"
+import { createLlamaPrompt } from "@/lib/createLlamaPrompt"
+
+import { getBase } from "./getBase"
+import { predict } from "./predict"
+
+export const getMusicTrack = async ({
+  game,
+  situation = "",
+  actionnable = "",
+  newDialogue = "",
+  newActionnables = [],
+}: {
+  game: Game;
+  situation: string;
+  actionnable: string;
+  newDialogue: string;
+  newActionnables: string[];
+}) => {
+  return ""
+}

src/app/queries/getSoundTrack.ts

@@ -0,0 +1,22 @@
+import { Game } from "@/app/games/types"
+import { createLlamaPrompt } from "@/lib/createLlamaPrompt"
+
+import { getBase } from "./getBase"
+import { predict } from "./predict"
+
+export const getSoundTrack = async ({
+  game,
+  situation = "",
+  actionnable = "",
+  newDialogue = "",
+  newActionnables = [],
+}: {
+  game: Game;
+  situation: string;
+  actionnable: string;
+  newDialogue: string;
+  newActionnables: string[];
+}) => {
+
+  return ""
+}

src/app/{predict.ts → queries/predict.ts}

@@ -7,6 +7,8 @@ const hf = hfi.endpoint(`${process.env.HF_INFERENCE_ENDPOINT_URL || ""}`)
 
 export async function predict(inputs: string) {
 
+  console.log(`predict: `, inputs)
+  
   let instructions = ""
   try {
     for await (const output of hf.textGenerationStream({

src/components/business/image-renderer.tsx

@@ -33,7 +33,9 @@ export const ImageRenderer = ({
         canvasRef.current = document.createElement('canvas');
         canvasRef.current.width = img.width;
         canvasRef.current.height = img.height;
-        contextRef.current = canvasRef.current.getContext('2d');
+        contextRef.current = canvasRef.current.getContext('2d', {
+          willReadFrequently: true
+        });
         contextRef.current!.drawImage(img, 0, 0, img.width, img.height);
       }
       img.src = "data:image/png;base64," + maskBase64;
@@ -104,6 +106,14 @@ export const ImageRenderer = ({
       return false
     }
 
+    // sometimes we generate an image, but the segmentation fails
+    // so if we click anywhere bug there are no segments,
+    // we inform the rest of the app by passing nothing
+    if (isClickEvent && segments.length == 0) {
+      onUserAction("nothing, to trigger a scene reload")
+      return
+    }
+
     const boundingRect = imgRef.current!.getBoundingClientRect();
     const x = event.clientX - boundingRect.left;
     const y = event.clientY - boundingRect.top;
@@ -112,10 +122,12 @@ export const ImageRenderer = ({
 
     if (actionnable !== newSegment.label) {
       if (newSegment.label) {
-        console.log(`User is hovering "${newSegment.label}"`);
+        console.log(`User is hovering "${newSegment.label}"`)
       } else {
-        console.log(`Nothing in the area`);
+        console.log(`Nothing in the area`)
       }
+
+      // update the actionnable immediately, so we can show the hand / finger cursor pointer
       setActionnable(newSegment.label)
     }
 
@@ -126,7 +138,11 @@ export const ImageRenderer = ({
       console.log("User clicked on " + newSegment.label)
       onUserAction(actionnable)
     } else {
-      onUserHover(actionnable)
+      // only trigger hover events if there are segments,
+      // otherwise it's best to stay silent
+      if (segments.length) {
+        onUserHover(actionnable)
+      }
     }
   };
 
@@ -142,9 +158,9 @@ export const ImageRenderer = ({
         return
       }
 
-      console.log("still loading..")
+      // console.log("still loading..")
 
-      console.log("updating progress")
+      // console.log("updating progress")
       progress = progress + 1
       setProcessPercent(progress)
 

src/lib/parseJsonList.ts

@@ -0,0 +1,13 @@
+export function parseJsonList(content: string): string[] {
+  // Extract JSON array from the content
+  const start = content.indexOf("[");
+  const end = content.lastIndexOf("]");
+  const jsonContent = content.slice(start, end + 1);
+
+  // Parse as JSON into array of strings
+  let objects: string[] = [];
+  
+  objects = JSON.parse(jsonContent);
+
+  return objects;
+}