working on coordinate conversion

package-lock.json

@@ -11,6 +11,10 @@
         "@gorgonjs/gorgon": "^1.4.1",
         "@huggingface/agents": "^0.0.4",
         "@huggingface/inference": "^2.6.1",
+        "@photo-sphere-viewer/core": "^5.1.7",
+        "@photo-sphere-viewer/markers-plugin": "^5.1.7",
+        "@photo-sphere-viewer/video-plugin": "^5.1.7",
+        "@photo-sphere-viewer/virtual-tour-plugin": "^5.1.7",
         "@radix-ui/react-accordion": "^1.1.2",
         "@radix-ui/react-avatar": "^1.0.3",
         "@radix-ui/react-checkbox": "^1.0.4",
@@ -45,6 +49,7 @@
         "fs-extra": "^11.1.1",
         "lucide-react": "^0.260.0",
         "next": "13.4.10",
+        "photo-sphere-viewer-lensflare-plugin": "^1.1.1",
         "pick": "^0.0.1",
         "postcss": "8.4.26",
         "react": "18.2.0",

package.json

@@ -12,6 +12,10 @@
     "@gorgonjs/gorgon": "^1.4.1",
     "@huggingface/agents": "^0.0.4",
     "@huggingface/inference": "^2.6.1",
+    "@photo-sphere-viewer/core": "^5.1.7",
+    "@photo-sphere-viewer/markers-plugin": "^5.1.7",
+    "@photo-sphere-viewer/video-plugin": "^5.1.7",
+    "@photo-sphere-viewer/virtual-tour-plugin": "^5.1.7",
     "@radix-ui/react-accordion": "^1.1.2",
     "@radix-ui/react-avatar": "^1.0.3",
     "@radix-ui/react-checkbox": "^1.0.4",
@@ -46,6 +50,7 @@
     "fs-extra": "^11.1.1",
     "lucide-react": "^0.260.0",
     "next": "13.4.10",
+    "photo-sphere-viewer-lensflare-plugin": "^1.1.1",
     "pick": "^0.0.1",
     "postcss": "8.4.26",
     "react": "18.2.0",

src/app/engines.ts

@@ -47,8 +47,8 @@ export const engines: Record<string, Engine> = {
     label: "Spherical video",
     visible: false,
     enabled: false,
-    modelName: "",
-    modelUrl: "",
+    modelName: "Zeroscope",
+    modelUrl: "https://huggingface.co/cerspense/zeroscope_v2_576w",
   },
 
   spherical_stereogram: {

src/app/render.ts

@@ -29,7 +29,7 @@ export async function newRender({
     throw new Error(`cannot call the rendering API without actionnables, aborting..`)
   }
 
-  const nbFrames = engine.type === "cartesian_video" ? 8 : 1
+  const nbFrames = engine.type.includes("video") ? 8 : 1
 
   const cacheKey = `render/${JSON.stringify({ prompt, actionnables, nbFrames, type: engine.type })}`
 

src/components/business/renderer.tsx

@@ -194,7 +194,7 @@ export const Renderer = ({
               onEvent={handleMouseEvent}
               debug={debug}
             />
-          : engine.type === "spherical_image"
+          : (engine.type === "spherical_image" || engine.type === "spherical_video")
           ? <SphericalImage
               rendered={rendered}
               onEvent={handleMouseEvent}

src/components/business/spherical-image.tsx

@@ -1,8 +1,10 @@
-import { ReactPhotoSphereViewer } from "react-photo-sphere-viewer"
+import { useEffect, useRef, useState } from "react"
+import { PluginConstructor } from "@photo-sphere-viewer/core"
+import { LensflarePlugin, ReactPhotoSphereViewer } from "react-photo-sphere-viewer"
 
-import { SceneEventHandler } from "./types"
 import { RenderedScene } from "@/app/types"
-import { useEffect, useRef, useState } from "react"
+
+import { SceneEventHandler } from "./types"
 
 export function SphericalImage({
   rendered,
@@ -27,6 +29,7 @@ export function SphericalImage({
     defaultZoomLvl: 1,
     overlay: rendered.maskUrl || undefined,
     overlayOpacity: debug ? 0.5 : 0,
+    fisheye: false, // ..no!
     /*
     panoData: {
       fullWidth: 2000,
@@ -71,6 +74,20 @@ export function SphericalImage({
     task()
   }, [sceneConfig, rendered.assetUrl, ref.current])
 
+  const plugins: (PluginConstructor | [PluginConstructor, any])[] = [
+
+    // for the lensflare plugin we are also gonna need aw ay to determine the position
+    [LensflarePlugin, {
+      lensflares: [
+        {
+          id: 'sun',
+          position: { yaw: '145deg', pitch: '2deg' },
+          type: 0,
+        }
+      ]
+    }]
+  ]
+
   if (!rendered.assetUrl) {
     return null
   }
@@ -84,6 +101,7 @@ export function SphericalImage({
       // 
       height="100vh"
       width="100%"
+      // plugins={plugins}
 
       {...options}
    

src/lib/cartesianToSpherical.ts

@@ -0,0 +1,6 @@
+export function cartesianToSpherical(x: number, y: number, width: number, height: number): { yaw: number, pitch: number } {
+  const yaw = (x / width) * 360 - 180;
+  const pitch = (y / height) * 180 - 90;
+
+  return { yaw, pitch };
+}

src/lib/getCoordinatesFromMousePosition.ts

@@ -0,0 +1,39 @@
+export function getCoordinatesFromMousePosition({
+  mouseX,
+  mouseY,
+  containerWidth,
+  containerHeight,
+  currentYaw,
+  currentPitch,
+  imageWidth,
+  imageHeight,
+  fov,
+}: {
+  mouseX: number
+  mouseY: number
+  containerWidth: number
+  containerHeight: number
+  currentYaw: number
+  currentPitch: number
+  imageWidth: number
+  imageHeight: number
+  fov: number
+}): { x: number; y: number } {
+  // Convert mouse position to relative to the viewer's center
+  const relativeX = 2 * (mouseX / containerWidth) - 1;
+  const relativeY = 2 * (mouseY / containerHeight) - 1;
+
+  // Calculate angle differences (in degrees)
+  const deltaYaw = relativeX * fov / 2;
+  const deltaPitch = relativeY * fov / 2;
+
+  // Calculate new yaw and pitch
+  const newYaw = currentYaw + deltaYaw;
+  const newPitch = currentPitch + deltaPitch;
+
+  // Now convert these yaw, pitch back to (x, y) on the image
+  const x = ((newYaw + 180) / 360) * imageWidth;
+  const y = ((newPitch + 90) / 180) * imageHeight;
+
+  return { x, y };
+}

src/lib/sphericalToCartesian.ts

@@ -0,0 +1,6 @@
+function sphericalToCartesian(yaw: number, pitch: number, width: number, height: number): { x: number, y: number } {
+  const x = ((yaw + 180) / 360) * width;
+  const y = ((pitch + 90) / 180) * height;
+
+  return { x, y };
+}