feat: use video cover as thumbnail (#35)

<img width="1131" alt="image"
src="https://github.com/landing-ai/vision-agent-ui/assets/132986242/4c981213-59a1-47a3-831e-a0cf529ad865">

app/api/vision-agent/route.ts

@@ -1,8 +1,4 @@
-import {
-  StreamingTextResponse,
-  experimental_StreamData,
-  createStreamDataTransformer,
-} from 'ai';
+import { StreamingTextResponse } from 'ai';
 
 // import { auth } from '@/auth';
 import { MessageBase } from '../../../lib/types';
@@ -76,11 +72,15 @@ export const POST = withLogging(
         return new StreamingTextResponse(
           new ReadableStream({
             async start(controller) {
-              const { done, value } = await reader.read();
-              if (!done) {
-                const errorText = new TextDecoder().decode(value);
-                logger.error(session, errorText, request);
-                controller.error(new Error(`Response error: ${errorText}`));
+              try {
+                const { done, value } = await reader.read();
+                if (!done) {
+                  const errorText = new TextDecoder().decode(value);
+                  logger.error(session, errorText, request);
+                  controller.error(new Error(`Response error: ${errorText}`));
+                }
+              } catch (e) {
+                logger.error(session, (e as Error).message, request);
               }
             },
           }),
@@ -110,7 +110,6 @@ export const POST = withLogging(
     if (fetchResponse.body) {
       const encoder = new TextEncoder();
       const decoder = new TextDecoder();
-
       const stream = fetchResponse.body.pipeThrough(
         new TransformStream({
           transform: async (chunk, controller) => {

components/ui/Img.tsx

@@ -1,8 +1,10 @@
 'use client';
 
-import React from 'react';
+import React, { useEffect, useState } from 'react';
 import Image from 'next/image';
 import { cn } from '@/lib/utils';
+import { toast } from 'react-hot-toast';
+import { getVideoCover } from '@rajesh896/video-thumbnails-generator';
 
 const placeholder =
   'data:image/jpeg;base64,/9j/4AAQSkZJRgABAgEASABIAAD/4QDWRXhpZgAATU0AKgAAAAgABwEGAAMAAAABAAIAAAESAAMAAAABAAEAAAEaAAUAAAABAAAAYgEbAAUAAAABAAAAagEoAAMAAAABAAIAAAITAAMAAAABAAEAAIdpAAQAAAABAAAAcgAAAAAAAABIAAAAAQAAAEgAAAABAAeQAAAHAAAABDAyMjGRAQAHAAAABAECAwCgAAAHAAAABDAxMDCgAQADAAAAAQABAACgAgAEAAAAAQAAADKgAwAEAAAAAQAAADKkBgADAAAAAQAAAAAAAAAAAAD/wAARCAAyADIDASIAAhEBAxEB/8QAHwAAAQUBAQEBAQEAAAAAAAAAAAECAwQFBgcICQoL/8QAtRAAAgEDAwIEAwUFBAQAAAF9AQIDAAQRBRIhMUEGE1FhByJxFDKBkaEII0KxwRVS0fAkM2JyggkKFhcYGRolJicoKSo0NTY3ODk6Q0RFRkdISUpTVFVWV1hZWmNkZWZnaGlqc3R1dnd4eXqDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uHi4+Tl5ufo6erx8vP09fb3+Pn6/8QAHwEAAwEBAQEBAQEBAQAAAAAAAAECAwQFBgcICQoL/8QAtREAAgECBAQDBAcFBAQAAQJ3AAECAxEEBSExBhJBUQdhcRMiMoEIFEKRobHBCSMzUvAVYnLRChYkNOEl8RcYGRomJygpKjU2Nzg5OkNERUZHSElKU1RVVldYWVpjZGVmZ2hpanN0dXZ3eHl6goOEhYaHiImKkpOUlZaXmJmaoqOkpaanqKmqsrO0tba3uLm6wsPExcbHyMnK0tPU1dbX2Nna4uPk5ebn6Onq8vP09fb3+Pn6/9sAQwABAQEBAQECAQECAwICAgMEAwMDAwQGBAQEBAQGBwYGBgYGBgcHBwcHBwcHCAgICAgICQkJCQkLCwsLCwsLCwsL/9sAQwECAgIDAwMFAwMFCwgGCAsLCwsLCwsLCwsLCwsLCwsLCwsLCwsLCwsLCwsLCwsLCwsLCwsLCwsLCwsLCwsLCwsL/90ABAAE/9oADAMBAAIRAxEAPwD+nbSfDst/NnB5Ne5+G/htJMqsVOD610fw/wDCS3Lq7L3r6y0Lw7b20SgqMiluB4hpvwti2Asn51fufhVbsmQlfSCQQQjHFOIt24OKLAfEXiL4XtCjMiGvnrxH4SlsJG+UjFfqZf6Va3UZUgV84/ETwTGY3kRaVrbAfAp058//AK6T+zn/AM5/wr1uTw0BIwx3P+elM/4Roen+fyo5gP/Q/tp+G1vGIVJFer6n4lstGizM4XFeH/DzVUW3HPavmL9pX4sXvhmKV0Yqq5qbgfXeq/GjRLJipmH51yMn7Qvh2NtrTL+dfzYfGL9uqTw1PLGZjkE96+M7j/gotqtzf+WkrYz60rsD+1bwt8YtD12ZYYZlJPvXY+KlivdOMy8giv5q/wBjD9prxB8QNcgXezKxHOa/obsdZa48KRPMTuKiqA8dm09PNbjuaj/s9PSp5rtDM5B7nvUf2tfX9f8A69QB/9H+v3wLrW1RHuxmvJv2iPh43jLRZhEuSynpWN4X8Rm0lCs2BnivoKy1iy1W1ENyQcjFQB/J7+0x+x34yvtVmlsY3OScYBr418KfsLfES81xFmgkALDPBr+1rXPhR4d15jJJEj59qwtP+BXhiznE4t0BHsKEugH54fsJfsqS/Dqzt7m/TDgAnIr9j9Q1BNN0kW4OAq1yun2mk+G7fZAFG0dq868Y+LwyMit+FFtAEl1tfNb5x1NR/wBtr/fH+fwrxptbnLE5703+2p6QH//S/pMs+JjivYfDjv5Q5NePWn+uNev+HP8AVCkwPWdPdtvU1fnkk2n5j09aztP+7V6f7p+lLoBxeuu4jOCa8N1xmMjEnNe4a7/qzXh2t/6xqfQDlqKKKgD/2Q==';
@@ -18,9 +20,28 @@ const Img = React.forwardRef<
   });
   // const [isLoading, setIsLoading] = React.useState(true);
   const [_, startTransition] = React.useTransition();
+  const [thumbnail, setThumbnail] = useState<string>('');
+  const isVideo =
+    typeof src === 'string' ? src.toLowerCase().endsWith('.mp4') : false;
+
+  useEffect(() => {
+    if (!isVideo) {
+      return;
+    }
+    const generateThumbnail = async () => {
+      try {
+        const cover = await getVideoCover(src as string);
+        setThumbnail(cover);
+      } catch (e) {
+        toast.error((e as Error).message);
+      }
+    };
+    generateThumbnail();
+  }, [isVideo, src]);
+
   return (
     <Image
-      src={src}
+      src={isVideo ? thumbnail : src}
       placeholder={placeholder}
       width={dimensions.width}
       height={dimensions.height}

package.json

@@ -23,6 +23,7 @@
     "@radix-ui/react-slot": "^1.0.2",
     "@radix-ui/react-switch": "^1.0.3",
     "@radix-ui/react-tooltip": "^1.0.7",
+    "@rajesh896/video-thumbnails-generator": "^2.3.9",
     "@vercel/kv": "^1.0.1",
     "ai": "^2.2.31",
     "class-variance-authority": "^0.7.0",