Update app.py

app.py

@@ -81,7 +81,15 @@ def extract_frames(video_in, interval=24, output_format='.jpg'):
 
     return frames
 
+from transformers import AutoModelForCausalLM, CodeGenTokenizerFast as Tokenizer
+from PIL import Image
+
+cap_model_id = "vikhyatk/moondream1"
+cap_model = AutoModelForCausalLM.from_pretrained(cap_model_id, trust_remote_code=True)
+cap_tokenizer = Tokenizer.from_pretrained(cap_model_id)
+
 def process_image(image_in):
+    '''
     client = Client("https://vikhyatk-moondream1.hf.space/")
     result = client.predict(
 		image_in,	# filepath  in 'image' Image component
@@ -91,6 +99,12 @@ def process_image(image_in):
     )
     print(result)
     return result
+    '''
+    image = Image.open(image_in)
+    enc_image = cap_model.encode_image(image)
+    result = cap_model.answer_question(enc_image, "Describe precisely the image in one sentence.", cap_tokenizer)
+    print(result)
+    return result
 
 def extract_audio(video_path):
     video_clip = VideoFileClip(video_path)