Spaces:
Paused
Paused
File size: 4,025 Bytes
65ee86e 58490f9 65ee86e 7249a2e 65ee86e 1896b82 7249a2e 1896b82 65ee86e 7249a2e 1896b82 dc8d027 65ee86e dc8d027 65ee86e dc8d027 65ee86e 58490f9 65ee86e |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 |
"use client"
import { useCallback, useEffect, useRef, useState, useTransition } from "react"
import { useInterval } from "usehooks-ts"
import Webcam from "react-webcam"
import AutoSizer from "react-virtualized-auto-sizer"
import { see } from "./engine/see"
import { Progress } from "./interface/progress"
export function Observe({
onObserve,
}: {
onObserve: (observation: string, image: string) => void
}) {
const [_isPending, startTransition] = useTransition()
const [img, setImg] = useState<string>("")
const webcamRef = useRef<Webcam>(null)
const [isInitialized, setInitialized] = useState(false)
const [frameNumber, setFrameNumber] = useState(0)
const [isBusy, setBusy] = useState(false)
const [lastObservation, setLastObservation] = useState("Nothing to see yet.")
const [lastObservedAt, setLastObservedAt] = useState(Date.now())
const defaultWidth = 1280
const defaultHeight = 1024 // 720
// minimum wait time between calls
const minimumWaitTimeInSec = 10
// in case we need to record a video, check the last part of
// https://blog.openreplay.com/capture-real-time-images-and-videos-with-react-webcam/
const capture = useCallback(() => {
if (!webcamRef.current) { return }
const imageSrc = webcamRef.current.getScreenshot()
if (!imageSrc) { return }
setImg(imageSrc)
setFrameNumber(frameNumber + 1)
return imageSrc
}, [webcamRef])
// note: for some strange reason, the webcam (at least on macOS)
// has a "fade in effect", which means in the first few seconds,
// eg. if we capture at 800ms, if will be darker than normal
useEffect(() => {
if (webcamRef.current && img && !isInitialized) {
setInitialized(true)
}
}, [webcamRef.current, img, isInitialized])
const observe = () => {
if (isBusy) {
// console.log("we are already predicting: skippping turn")
return
}
const currentTimeInMs = Date.now()
const elapsedTimeInMs = currentTimeInMs - lastObservedAt
const elapsedTimeInSec = elapsedTimeInMs / 1000
if (elapsedTimeInSec < minimumWaitTimeInSec) {
// console.log("minimum wait time between calls not reached: skipping turn")
return
}
setBusy(true)
// console.log("Capturing new frame from webcam..")
startTransition(async () => {
const imageBase64 = capture()
if (!imageBase64) {
console.log("Failed to capture a new frame")
setTimeout(() => {
setBusy(false)
setLastObservedAt(Date.now())
}, 2000)
return
}
const prompt = `What do you see here?`
console.log("JULIAN: disabled watch")
// console.log("Calling IDEFICS..")
const newObservation = await see({ prompt, imageBase64 })
// console.log("New observation: ", newObservation)
if (newObservation && newObservation !== lastObservation) {
// console.log("update!")
setLastObservation(newObservation || "")
onObserve(newObservation || "", imageBase64)
}
setLastObservedAt(Date.now())
// comment to disable the infinite loop!
setBusy(false)
})
// console.log("observation ended!")
}
useInterval(() => {
observe()
}, 1000)
return (
<AutoSizer>
{({ height, width }) => (
<>
<Webcam
ref={webcamRef}
className="fixed top-0 left-0 right-0 w-screen"
screenshotFormat='image/jpeg'
// screenshotFormat="image/webp"
mirrored={false}
videoConstraints={{
width: { min: defaultWidth },
height: { min: defaultHeight },
aspectRatio: defaultWidth / defaultHeight,
facingMode: "user",
// if the device allows it, we can use the back camera
// facingMode: { exact: "environment" }
} as MediaTrackConstraints}
/>
<Progress
isLoading={isBusy}
resetKey=""
className="right-6"
/>
</>
)}
</AutoSizer>
)
}
|