initial commit

.gitattributes

@@ -0,0 +1,38 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+football.mp4 filter=lfs diff=lfs merge=lfs -text
+safari2.mp4 filter=lfs diff=lfs merge=lfs -text
+cat.mp4 filter=lfs diff=lfs merge=lfs -text

README copy.md

@@ -0,0 +1,15 @@
+---
+title: Omdet Turbo Open Vocabulary
+emoji: 📹
+colorFrom: red
+colorTo: blue
+sdk: gradio
+sdk_version: 4.42.0
+app_file: app.py
+pinned: false
+license: apache-2.0
+short_description: Video captioning/open-vocabulary/zero-shot
+
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

README.md

@@ -0,0 +1,13 @@
+---
+title: RTDETR Compiled Speed Test
+emoji: 🐠
+colorFrom: purple
+colorTo: yellow
+sdk: gradio
+sdk_version: 4.44.0
+app_file: app.py
+pinned: false
+license: apache-2.0
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -0,0 +1,178 @@
+import os
+import time
+
+import gradio as gr
+import numpy as np
+import requests
+import spaces
+import supervision as sv
+import torch
+from PIL import Image
+from tqdm import tqdm
+
+from transformers import AutoModelForObjectDetection, AutoProcessor
+
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+processor = AutoProcessor.from_pretrained("PekingU/rtdetr_r50vd_coco_o365")
+model = AutoModelForObjectDetection.from_pretrained(
+    "PekingU/rtdetr_r50vd_coco_o365",
+    disable_custom_kernels=True,
+    torch_dtype=torch.float16,
+).to(device)
+model_compiled = torch.compile(model, mode="reduce-overhead")
+
+url = "http://images.cocodataset.org/val2017/000000039769.jpg"
+image = Image.open(requests.get(url, stream=True).raw)
+inputs = processor(images=image, return_tensors="pt").to("cuda").to(torch.float16)
+
+print("Compiling model...")
+start_time = time.time()
+with torch.no_grad():
+    for _ in range(10):
+        outputs = model_compiled(**inputs)
+        _ = outputs[0].cpu()
+print(f"Model compiled in {time.time() - start_time:.2f} seconds.")
+
+css = """
+.feedback textarea {font-size: 24px !important}
+"""
+
+BOUNDING_BOX_ANNOTATOR = sv.BoundingBoxAnnotator()
+MASK_ANNOTATOR = sv.MaskAnnotator()
+LABEL_ANNOTATOR = sv.LabelAnnotator()
+TRACKER = sv.ByteTrack()
+
+
+def calculate_end_frame_index(source_video_path):
+    video_info = sv.VideoInfo.from_video_path(source_video_path)
+    return min(video_info.total_frames, video_info.fps * 5)
+
+
+def annotate_image(input_image, detections, labels) -> np.ndarray:
+    output_image = MASK_ANNOTATOR.annotate(input_image, detections)
+    output_image = BOUNDING_BOX_ANNOTATOR.annotate(output_image, detections)
+    output_image = LABEL_ANNOTATOR.annotate(output_image, detections, labels=labels)
+    return output_image
+
+
+@spaces.GPU
+def process_video(
+    input_video,
+    confidence_threshold,
+    max_side,
+    progress=gr.Progress(track_tqdm=True),
+):
+    video_info = sv.VideoInfo.from_video_path(input_video)
+    total = calculate_end_frame_index(input_video)
+    frame_generator = sv.get_video_frames_generator(source_path=input_video, end=total)
+
+    result_file_name = "output.mp4"
+    result_file_path = os.path.join(os.getcwd(), result_file_name)
+    all_fps = []
+    with sv.VideoSink(result_file_path, video_info=video_info) as sink:
+        for _ in tqdm(range(total), desc="Processing video.."):
+            try:
+                frame = next(frame_generator)
+            except StopIteration:
+                break
+            results, fps = query(frame, confidence_threshold, max_side=max_side)
+            all_fps.append(fps)
+            final_labels = []
+            detections = []
+
+            detections = sv.Detections.from_transformers(results[0])
+            detections = TRACKER.update_with_detections(detections)
+            for label in detections.class_id.tolist():
+                final_labels.append(model.config.id2label[label])
+            frame = annotate_image(
+                input_image=frame,
+                detections=detections,
+                labels=final_labels,
+            )
+            sink.write_frame(frame)
+
+    avg_fps = np.mean(all_fps)
+    return result_file_path, gr.Markdown(
+        f'<h3 style="text-align: center;">Model inference FPS: {avg_fps:.2f}</h3>',
+        visible=True,
+    )
+
+
+def query(frame, confidence_threshold, max_side=640):
+    frame_resized = sv.resize_image(
+        image=frame, resolution_wh=(max_side, max_side), keep_aspect_ratio=True
+    )
+    image = Image.fromarray(frame_resized)
+    inputs = processor(images=image, return_tensors="pt").to(device, torch.float16)
+    with torch.no_grad():
+        start = time.time()
+        outputs = model_compiled(**inputs)
+        outputs[0].cpu()
+        fps = 1 / (time.time() - start)
+    target_sizes = torch.tensor([frame.shape[:2]]).to(device)
+
+    results = processor.post_process_object_detection(
+        outputs=outputs,
+        threshold=confidence_threshold,
+        target_sizes=target_sizes,
+    )
+    return results, fps
+
+
+with gr.Blocks(theme=gr.themes.Soft(), css=css) as demo:
+    gr.Markdown("## Real Time Object Detection with compiled RT-DETR")
+    gr.Markdown(
+        """
+        This is a demo for real-time object detection using RT-DETR compiled.<br>
+        It runs on ZeroGPU which captures GPU every first time you infer.<br>
+        This combined with video processing time means that the demo inference time is slower than the model's actual inference time.<br>
+        The actual model average inference FPS is displayed under the processed video after inference.
+        """
+    )
+    gr.Markdown(
+        "Simply upload a video! You can also play with confidence threshold or try the examples below. 👇"
+    )
+
+    with gr.Row():
+        with gr.Column():
+            input_video = gr.Video(label="Input Video")
+        with gr.Column():
+            output_video = gr.Video(label="Output Video (5s max)")
+            actual_fps = gr.Markdown("", visible=False)
+    with gr.Row():
+        conf = gr.Slider(
+            label="Confidence Threshold",
+            minimum=0.1,
+            maximum=1.0,
+            value=0.3,
+            step=0.05,
+        )
+        max_side = gr.Slider(
+            label="Image Size",
+            minimum=240,
+            maximum=1080,
+            value=640,
+            step=10,
+        )
+    with gr.Row():
+        submit = gr.Button(variant="primary")
+
+    example = gr.Examples(
+        examples=[
+            ["./football.mp4", 0.3, 640],
+            ["./cat.mp4", 0.3, 640],
+            ["./safari2.mp4", 0.3, 640],
+        ],
+        inputs=[input_video, conf, max_side],
+        outputs=output_video,
+    )
+
+    submit.click(
+        fn=process_video,
+        inputs=[input_video, conf, max_side],
+        outputs=[output_video, actual_fps],
+    )
+
+if __name__ == "__main__":
+    demo.launch(show_error=True)

cat.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:07539c031a516acecf58b8751f74ba90182efe4c4ad25513038f10564739eadd
+size 810095

football.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:56a85c5c7d5d6e0825f76a71e5e3ee2ce35c8ffbe841ef4bfa544af1089259aa
+size 2855852

requirements.txt

@@ -0,0 +1,7 @@
+torch
+timm
+requests
+numpy==1.26.3
+git+https://github.com/yonigozlan/transformers.git@optim-rt-detr
+supervision
+spaces

safari2.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1c7f26f775768d06219b19acb4c071e40928f1042b7b4fa2d876095c72139e19
+size 3011687