|
import gradio as gr |
|
import subprocess |
|
import cv2 |
|
import torch |
|
import torchaudio |
|
from torchvision.models.detection import fasterrcnn_resnet50_fpn |
|
import torchvision.transforms as transforms |
|
from PIL import Image |
|
from transformers import Wav2Vec2ForCTC, Wav2Vec2Processor |
|
|
|
class FasterRCNNDetector: |
|
def __init__(self): |
|
self.model = fasterrcnn_resnet50_fpn(pretrained=True) |
|
self.model.eval() |
|
self.classes = [ |
|
"__background__", "person", "bicycle", "car", "motorcycle", "airplane", "bus", |
|
"train", "truck", "boat", "traffic light", "fire hydrant", "N/A", "stop sign", |
|
"parking meter", "bench", "bird", "cat", "dog", "horse", "sheep", "cow", |
|
"elephant", "bear", "zebra", "giraffe", "N/A", "backpack", "umbrella", "N/A", "N/A", |
|
"handbag", "tie", "suitcase", "frisbee", "skis", "snowboard", "sports ball", |
|
"kite", "baseball bat", "baseball glove", "skateboard", "surfboard", "tennis racket", |
|
"bottle", "N/A", "wine glass", "cup", "fork", "knife", "spoon", "bowl", |
|
"banana", "apple", "sandwich", "orange", "broccoli", "carrot", "hot dog", "pizza", |
|
"donut", "cake", "chair", "couch", "potted plant", "bed", "N/A", "dining table", |
|
"N/A", "N/A", "toilet", "N/A", "tv", "laptop", "mouse", "remote", "keyboard", "cell phone", |
|
"microwave", "oven", "toaster", "sink", "refrigerator", "N/A", "book", |
|
"clock", "vase", "scissors", "teddy bear", "hair drier", "toothbrush" |
|
] |
|
|
|
def detect_objects(self, image): |
|
image_pil = Image.fromarray(image) |
|
transform = transforms.Compose([transforms.ToTensor()]) |
|
image_tensor = transform(image_pil).unsqueeze(0) |
|
|
|
with torch.no_grad(): |
|
prediction = self.model(image_tensor) |
|
|
|
boxes = prediction[0]['boxes'] |
|
labels = prediction[0]['labels'] |
|
scores = prediction[0]['scores'] |
|
|
|
for box, label, score in zip(boxes, labels, scores): |
|
box = [int(i) for i in box] |
|
cv2.rectangle(image, (box[0], box[1]), (box[2], box[3]), (0, 255, 0), 2) |
|
cv2.putText(image, self.classes[label], (box[0], box[1] - 10), cv2.FONT_HERSHEY_SIMPLEX, 0.9, (36,255,12), 2) |
|
|
|
return image |
|
|
|
class JarvisModels: |
|
def __init__(self): |
|
self.processor = Wav2Vec2Processor.from_pretrained("facebook/wav2vec2-base-960h") |
|
self.model = Wav2Vec2ForCTC.from_pretrained("facebook/wav2vec2-base-960h") |
|
|
|
async def generate_response(self, prompt): |
|
|
|
response = gr.Interface.load("models/openai-community/gpt2").process(prompt) |
|
return response |
|
|
|
async def transcribe_audio(self, audio_file): |
|
input_audio, _ = torchaudio.load(audio_file) |
|
input_values = self.processor(input_audio, return_tensors="pt").input_values |
|
logits = self.model(input_values).logits |
|
predicted_ids = torch.argmax(logits, dim=-1) |
|
transcription = self.processor.batch_decode(predicted_ids) |
|
return transcription[0] |
|
|
|
def transcribe(audio): |
|
global messages |
|
|
|
audio_file = open(audio, "rb") |
|
|
|
transcript = "Lorem ipsum dolor sit amet, consectetur adipiscing elit." |
|
|
|
|
|
system_message = {"role": "system", "content": "Lorem ipsum dolor sit amet, consectetur adipiscing elit."} |
|
|
|
subprocess.call(["say", system_message['content']]) |
|
|
|
chat_transcript = "User: " + transcript + "\n\n" + "System: " + system_message['content'] + "\n\n" |
|
|
|
return chat_transcript |
|
|
|
detector = FasterRCNNDetector() |
|
|
|
iface = gr.Interface( |
|
fn=[detector.detect_objects, JarvisModels().transcribe_audio, JarvisModels().generate_response, transcribe], |
|
inputs=[ |
|
gr.inputs.Video(label="Webcam", parameters={"fps": 30}), |
|
gr.inputs.Audio(source="microphone", type="filepath") |
|
], |
|
outputs=[ |
|
gr.outputs.Image(), |
|
"text", |
|
"text", |
|
"text" |
|
], |
|
title="Vision and Speech Interface", |
|
description="This interface detects objects in the webcam feed and transcribes speech recorded through the microphone." |
|
) |
|
iface.launch() |
|
|
