| import cv2 | |
| import gradio as gr | |
| import edge_tts | |
| import tempfile | |
| import numpy as np | |
| from torchvision.models.detection import fasterrcnn_resnet50_fpn | |
| import torchvision.transforms as transforms | |
| from PIL import Image | |
| from huggingface_hub import InferenceClient | |
| class YoloDetector: | |
| def __init__(self, weights_path, cfg_path, names_path): | |
| self.net = cv2.dnn.readNet(weights_path, cfg_path) | |
| self.classes = [] | |
| with open(names_path, "r") as f: | |
| self.classes = [line.strip() for line in f.readlines()] | |
| self.layer_names = self.net.getLayerNames() | |
| self.output_layers = [self.layer_names[i[0] - 1] for i in self.net.getUnconnectedOutLayers()] | |
| def detect_objects(self, frame): | |
| height, width, channels = frame.shape | |
| blob = cv2.dnn.blobFromImage(frame, 0.00392, (416, 416), (0, 0, 0), True, crop=False) | |
| self.net.setInput(blob) | |
| outs = self.net.forward(self.output_layers) | |
| class_ids = [] | |
| confidences = [] | |
| boxes = [] | |
| for out in outs: | |
| for detection in out: | |
| scores = detection[5:] | |
| class_id = np.argmax(scores) | |
| confidence = scores[class_id] | |
| if confidence > 0.5: | |
| center_x = int(detection[0] * width) | |
| center_y = int(detection[1] * height) | |
| w = int(detection[2] * width) | |
| h = int(detection[3] * height) | |
| x = int(center_x - w / 2) | |
| y = int(center_y - h / 2) | |
| boxes.append([x, y, w, h]) | |
| confidences.append(float(confidence)) | |
| class_ids.append(class_id) | |
| indexes = cv2.dnn.NMSBoxes(boxes, confidences, 0.5, 0.4) | |
| font = cv2.FONT_HERSHEY_PLAIN | |
| for i in range(len(boxes)): | |
| if i in indexes: | |
| x, y, w, h = boxes[i] | |
| label = str(self.classes[class_ids[i]]) | |
| color = (0, 255, 0) | |
| cv2.rectangle(frame, (x, y), (x + w, y + h), color, 2) | |
| cv2.putText(frame, label, (x, y + 30), font, 3, color, 2) | |
| return frame | |
| class JarvisModels: | |
| def __init__(self): | |
| self.client1 = InferenceClient("mistralai/Mixtral-8x7B-Instruct-v0.1") | |
| self.detector = YoloDetector("yolov3.weights", "yolov3.cfg", "coco.names") | |
| async def generate_model1(self, prompt): | |
| generate_kwargs = dict( | |
| temperature=0.6, | |
| max_new_tokens=256, | |
| top_p=0.95, | |
| repetition_penalty=1, | |
| do_sample=True, | |
| seed=42, | |
| ) | |
| formatted_prompt = system_instructions1 + prompt + "[JARVIS]" | |
| stream = self.client1.text_generation( | |
| formatted_prompt, **generate_kwargs, stream=True, details=True, return_full_text=True) | |
| output = "" | |
| for response in stream: | |
| output += response.token.text | |
| communicate = edge_tts.Communicate(output) | |
| with tempfile.NamedTemporaryFile(delete=False, suffix=".wav") as tmp_file: | |
| tmp_path = tmp_file.name | |
| communicate.save(tmp_path) | |
| return tmp_path | |
| class FasterRCNNDetector: | |
| def __init__(self): | |
| self.model = fasterrcnn_resnet50_fpn(pretrained=True) | |
| self.model.eval() | |
| self.classes = [ | |
| "__background__", "person", "bicycle", "car", "motorcycle", "airplane", "bus", | |
| "train", "truck", "boat", "traffic light", "fire hydrant", "N/A", "stop sign", | |
| "parking meter", "bench", "bird", "cat", "dog", "horse", "sheep", "cow", | |
| "elephant", "bear", "zebra", "giraffe", "N/A", "backpack", "umbrella", "N/A", "N/A", | |
| "handbag", "tie", "suitcase", "frisbee", "skis", "snowboard", "sports ball", | |
| "kite", "baseball bat", "baseball glove", "skateboard", "surfboard", "tennis racket", | |
| "bottle", "N/A", "wine glass", "cup", "fork", "knife", "spoon", "bowl", | |
| "banana", "apple", "sandwich", "orange", "broccoli", "carrot", "hot dog", "pizza", | |
| "donut", "cake", "chair", "couch", "potted plant", "bed", "N/A", "dining table", | |
| "N/A", "N/A", "toilet", "N/A", "tv", "laptop", "mouse", "remote", "keyboard", "cell phone", | |
| "microwave", "oven", "toaster", "sink", "refrigerator", "N/A", "book", | |
| "clock", "vase", "scissors", "teddy bear", "hair drier", "toothbrush" | |
| ] | |
| def detect_objects(self, image): | |
| image_pil = Image.fromarray(image) | |
| transform = transforms.Compose([transforms.ToTensor()]) | |
| image_tensor = transform(image_pil).unsqueeze(0) | |
| with torch.no_grad(): | |
| prediction = self.model(image_tensor) | |
| boxes = prediction[0]['boxes'] | |
| labels = prediction[0]['labels'] | |
| scores = prediction[0]['scores'] | |
| for box, label, score in zip(boxes, labels, scores): | |
| box = [int(i) for i in box] | |
| cv2.rectangle(image, (box[0], box[1]), (box[2], box[3]), (0, 255, 0), 2) | |
| cv2.putText(image, self.classes[label], (box[0], box[1] - 10), cv2.FONT_HERSHEY_SIMPLEX, 0.9, (36,255,12), 2) | |
| return image | |
| def generate_response(frame): | |
| jarvis = JarvisModels() | |
| detector = FasterRCNNDetector() | |
| frame_with_boxes = jarvis.detector.detect_objects(frame) | |
| cv2.imwrite("temp.jpg", frame_with_boxes) | |
| communicate = edge_tts.Communicate("Objects detected!") | |
| with tempfile.NamedTemporaryFile(delete=False, suffix=".wav") as tmp_file: | |
| tmp_path = tmp_file.name | |
| communicate.save(tmp_path) | |
| return tmp_path | |
| iface = gr.Webcam(gr.Video(label="Webcam", parameters=["fps=30"], is_streaming=True), preprocess=generate_response, postprocess=FasterRCNNDetector().detect_objects, show_loading=False) | |
| gr.Interface(fn=iface, layout="vertical", capture_session=True).launch() |