apratim24 commited on
Commit
f96d4eb
·
verified ·
1 Parent(s): ad36fb3

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +55 -4
app.py CHANGED
@@ -1,5 +1,6 @@
1
  import gradio as gr
2
  from PIL import Image, ImageDraw, ImageFont
 
3
 
4
 
5
  # Use a pipeline as a high-level helper
@@ -8,11 +9,58 @@ from transformers import pipeline
8
  # model_path = ("../Models/models--facebook--detr-resnet-50/snapshots"
9
  # "/1d5f47bd3bdd2c4bbfa585418ffe6da5028b4c0b")
10
 
 
 
 
11
  object_detector = pipeline("object-detection",
12
  model="facebook/detr-resnet-50")
13
 
14
- # object_detector = pipeline("object-detection",
15
- # model=model_path)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
16
 
17
 
18
  def draw_bounding_boxes(image, detections, font_path=None, font_size=50):
@@ -61,7 +109,10 @@ def detect_object(image):
61
  raw_image = image
62
  output = object_detector(raw_image)
63
  processed_image = draw_bounding_boxes(raw_image, output)
64
- return processed_image
 
 
 
65
 
66
  examples = [
67
  ["example1.jpg"],
@@ -74,7 +125,7 @@ demo = gr.Interface(fn=detect_object,
74
  outputs=[gr.Image(label="Processed Image", type="pil")],
75
  examples = examples,
76
  title="Object Detector",
77
- description="Detect objects in the input image with bounding boxes.")
78
  demo.launch()
79
 
80
 
 
1
  import gradio as gr
2
  from PIL import Image, ImageDraw, ImageFont
3
+ import scipy.io.wavfile as wavfile
4
 
5
 
6
  # Use a pipeline as a high-level helper
 
9
  # model_path = ("../Models/models--facebook--detr-resnet-50/snapshots"
10
  # "/1d5f47bd3bdd2c4bbfa585418ffe6da5028b4c0b")
11
 
12
+ # object_detector = pipeline("object-detection",
13
+ # model=model_path)
14
+
15
  object_detector = pipeline("object-detection",
16
  model="facebook/detr-resnet-50")
17
 
18
+ narrator = pipeline("text-to-speech",
19
+ model="kakao-enterprise/vits-ljs")
20
+
21
+
22
+ # Define the function to generate audio from text
23
+ def generate_audio(text):
24
+ # Generate the narrated text
25
+ narrated_text = narrator(text)
26
+
27
+ # Save the audio to a WAV file
28
+ wavfile.write("output.wav", rate=narrated_text["sampling_rate"],
29
+ data=narrated_text["audio"][0])
30
+
31
+ # Return the path to the saved audio file
32
+ return "output.wav"
33
+
34
+
35
+
36
+ def read_objects(detection_objects):
37
+ # Initialize counters for each object label
38
+ object_counts = {}
39
+
40
+ # Count the occurrences of each label
41
+ for detection in detection_objects:
42
+ label = detection['label']
43
+ if label in object_counts:
44
+ object_counts[label] += 1
45
+ else:
46
+ object_counts[label] = 1
47
+
48
+ # Generate the response string
49
+ response = "This picture contains"
50
+ labels = list(object_counts.keys())
51
+ for i, label in enumerate(labels):
52
+ response += f" {object_counts[label]} {label}"
53
+ if object_counts[label] > 1:
54
+ response += "s"
55
+ if i < len(labels) - 2:
56
+ response += ","
57
+ elif i == len(labels) - 2:
58
+ response += " and"
59
+
60
+ response += "."
61
+
62
+ return response
63
+
64
 
65
 
66
  def draw_bounding_boxes(image, detections, font_path=None, font_size=50):
 
109
  raw_image = image
110
  output = object_detector(raw_image)
111
  processed_image = draw_bounding_boxes(raw_image, output)
112
+ natural_text = read_objects(output)
113
+ processed_audio = generate_audio(natural_text)
114
+ return processed_image, processed_audio
115
+
116
 
117
  examples = [
118
  ["example1.jpg"],
 
125
  outputs=[gr.Image(label="Processed Image", type="pil")],
126
  examples = examples,
127
  title="Object Detector",
128
+ description="Detect objects in the input image with bounding boxes with audio description.")
129
  demo.launch()
130
 
131