Spaces:
Sleeping
Sleeping
Update app.py
Browse files
app.py
CHANGED
@@ -1,5 +1,6 @@
|
|
1 |
import gradio as gr
|
2 |
from PIL import Image, ImageDraw, ImageFont
|
|
|
3 |
|
4 |
|
5 |
# Use a pipeline as a high-level helper
|
@@ -8,11 +9,58 @@ from transformers import pipeline
|
|
8 |
# model_path = ("../Models/models--facebook--detr-resnet-50/snapshots"
|
9 |
# "/1d5f47bd3bdd2c4bbfa585418ffe6da5028b4c0b")
|
10 |
|
|
|
|
|
|
|
11 |
object_detector = pipeline("object-detection",
|
12 |
model="facebook/detr-resnet-50")
|
13 |
|
14 |
-
|
15 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
16 |
|
17 |
|
18 |
def draw_bounding_boxes(image, detections, font_path=None, font_size=50):
|
@@ -61,7 +109,10 @@ def detect_object(image):
|
|
61 |
raw_image = image
|
62 |
output = object_detector(raw_image)
|
63 |
processed_image = draw_bounding_boxes(raw_image, output)
|
64 |
-
|
|
|
|
|
|
|
65 |
|
66 |
examples = [
|
67 |
["example1.jpg"],
|
@@ -74,7 +125,7 @@ demo = gr.Interface(fn=detect_object,
|
|
74 |
outputs=[gr.Image(label="Processed Image", type="pil")],
|
75 |
examples = examples,
|
76 |
title="Object Detector",
|
77 |
-
description="Detect objects in the input image with bounding boxes.")
|
78 |
demo.launch()
|
79 |
|
80 |
|
|
|
1 |
import gradio as gr
|
2 |
from PIL import Image, ImageDraw, ImageFont
|
3 |
+
import scipy.io.wavfile as wavfile
|
4 |
|
5 |
|
6 |
# Use a pipeline as a high-level helper
|
|
|
9 |
# model_path = ("../Models/models--facebook--detr-resnet-50/snapshots"
|
10 |
# "/1d5f47bd3bdd2c4bbfa585418ffe6da5028b4c0b")
|
11 |
|
12 |
+
# object_detector = pipeline("object-detection",
|
13 |
+
# model=model_path)
|
14 |
+
|
15 |
object_detector = pipeline("object-detection",
|
16 |
model="facebook/detr-resnet-50")
|
17 |
|
18 |
+
narrator = pipeline("text-to-speech",
|
19 |
+
model="kakao-enterprise/vits-ljs")
|
20 |
+
|
21 |
+
|
22 |
+
# Define the function to generate audio from text
|
23 |
+
def generate_audio(text):
|
24 |
+
# Generate the narrated text
|
25 |
+
narrated_text = narrator(text)
|
26 |
+
|
27 |
+
# Save the audio to a WAV file
|
28 |
+
wavfile.write("output.wav", rate=narrated_text["sampling_rate"],
|
29 |
+
data=narrated_text["audio"][0])
|
30 |
+
|
31 |
+
# Return the path to the saved audio file
|
32 |
+
return "output.wav"
|
33 |
+
|
34 |
+
|
35 |
+
|
36 |
+
def read_objects(detection_objects):
|
37 |
+
# Initialize counters for each object label
|
38 |
+
object_counts = {}
|
39 |
+
|
40 |
+
# Count the occurrences of each label
|
41 |
+
for detection in detection_objects:
|
42 |
+
label = detection['label']
|
43 |
+
if label in object_counts:
|
44 |
+
object_counts[label] += 1
|
45 |
+
else:
|
46 |
+
object_counts[label] = 1
|
47 |
+
|
48 |
+
# Generate the response string
|
49 |
+
response = "This picture contains"
|
50 |
+
labels = list(object_counts.keys())
|
51 |
+
for i, label in enumerate(labels):
|
52 |
+
response += f" {object_counts[label]} {label}"
|
53 |
+
if object_counts[label] > 1:
|
54 |
+
response += "s"
|
55 |
+
if i < len(labels) - 2:
|
56 |
+
response += ","
|
57 |
+
elif i == len(labels) - 2:
|
58 |
+
response += " and"
|
59 |
+
|
60 |
+
response += "."
|
61 |
+
|
62 |
+
return response
|
63 |
+
|
64 |
|
65 |
|
66 |
def draw_bounding_boxes(image, detections, font_path=None, font_size=50):
|
|
|
109 |
raw_image = image
|
110 |
output = object_detector(raw_image)
|
111 |
processed_image = draw_bounding_boxes(raw_image, output)
|
112 |
+
natural_text = read_objects(output)
|
113 |
+
processed_audio = generate_audio(natural_text)
|
114 |
+
return processed_image, processed_audio
|
115 |
+
|
116 |
|
117 |
examples = [
|
118 |
["example1.jpg"],
|
|
|
125 |
outputs=[gr.Image(label="Processed Image", type="pil")],
|
126 |
examples = examples,
|
127 |
title="Object Detector",
|
128 |
+
description="Detect objects in the input image with bounding boxes with audio description.")
|
129 |
demo.launch()
|
130 |
|
131 |
|