Spaces:
Running
Running
Create app.py
Browse files
app.py
ADDED
@@ -0,0 +1,114 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
import os
|
2 |
+
import gradio as gr
|
3 |
+
import pandas as pd
|
4 |
+
from google import genai
|
5 |
+
|
6 |
+
# Client and prompt setup
|
7 |
+
client = genai.Client(api_key=os.getenv('GOOGLE_API_KEY'))
|
8 |
+
model_name = "gemini-2.0-flash-exp" # Change to other models, but be careful as response might be with different structure
|
9 |
+
safety_settings = [
|
10 |
+
genai.types.SafetySetting(
|
11 |
+
category="HARM_CATEGORY_DANGEROUS_CONTENT",
|
12 |
+
threshold="BLOCK_ONLY_HIGH",
|
13 |
+
),
|
14 |
+
]
|
15 |
+
bounding_box_system_instructions = """Return bounding boxes as a JSON array with labels, CO2 estimate, and an explanation. Never return masks or code fencing. Limit to 5 objects."""
|
16 |
+
prompt = """Provide an estimation of how much CO2 is involved in all activities in this picture. Give CO2 in grams.
|
17 |
+
|
18 |
+
As examples, think of transport, smoking, meat, and other similar emission activities.
|
19 |
+
Do not provide actions that don't have CO2 emissions.
|
20 |
+
|
21 |
+
Be comprehensive, but don't list more than 10 objects. Detect the 2D bounding boxes of these activities,
|
22 |
+
including the label, the CO2 gram quantity, and a short explanation explaining the estimation
|
23 |
+
for each activity.
|
24 |
+
"""
|
25 |
+
|
26 |
+
def parse_json(json_output):
|
27 |
+
# Based on https://github.com/google-gemini/cookbook/blob/main/gemini-2/spatial_understanding.ipynb
|
28 |
+
lines = json_output.splitlines()
|
29 |
+
for i, line in enumerate(lines):
|
30 |
+
if line == "```json":
|
31 |
+
json_output = "\n".join(lines[i+1:]) # Remove everything before "```json"
|
32 |
+
json_output = json_output.split("```")[0] # Remove everything after the closing "```"
|
33 |
+
break # Exit the loop once "```json" is found
|
34 |
+
return json_output
|
35 |
+
|
36 |
+
def parse_info(image, json_data):
|
37 |
+
width, height = image.size
|
38 |
+
df_data = []
|
39 |
+
boxes_with_labels = []
|
40 |
+
|
41 |
+
if not isinstance(json_data, list):
|
42 |
+
logging.error("JSON data is not a list as expected.")
|
43 |
+
return boxes_with_labels, pd.DataFrame(df_data)
|
44 |
+
|
45 |
+
# Iterate over each detected action actions
|
46 |
+
for action in info:
|
47 |
+
box_2d = action.get("box_2d")
|
48 |
+
label = action.get("label")
|
49 |
+
co2_grams = action.get("co2_grams")
|
50 |
+
explanation = action.get("explanation")
|
51 |
+
|
52 |
+
if not all([box_2d, label, co2_grams, explanation]):
|
53 |
+
continue
|
54 |
+
|
55 |
+
# Convert normalized coordinates to absolute coordinates
|
56 |
+
abs_y1 = int(box_2d[0] / 1000 * height)
|
57 |
+
abs_x1 = int(box_2d[1] / 1000 * width)
|
58 |
+
abs_y2 = int(box_2d[2] / 1000 * height)
|
59 |
+
abs_x2 = int(box_2d[3] / 1000 * width)
|
60 |
+
|
61 |
+
abs_x1, abs_x2 = min(abs_x1, abs_x2), max(abs_x1, abs_x2)
|
62 |
+
abs_y1, abs_y2 = min(abs_y1, abs_y2), max(abs_y1, abs_y2)
|
63 |
+
|
64 |
+
boxes_with_labels.append([(abs_x1, abs_y1, abs_x2, abs_y2), label])
|
65 |
+
|
66 |
+
df_data.append({
|
67 |
+
"label": label,
|
68 |
+
"co2": co2_grams,
|
69 |
+
"explanation": explanation
|
70 |
+
})
|
71 |
+
|
72 |
+
return boxes_with_labels, pd.DataFrame(df_data)
|
73 |
+
|
74 |
+
def estimate_co2(image):
|
75 |
+
resized_image = image.resize(
|
76 |
+
(1024, int(1024 * img.size[1] / img.size[0])),
|
77 |
+
Image.Resampling.LANCZOS
|
78 |
+
)
|
79 |
+
|
80 |
+
# Get resuls from model
|
81 |
+
response = client.models.generate_content(
|
82 |
+
model=model_name,
|
83 |
+
contents=[prompt, resized_image],
|
84 |
+
config = genai.types.GenerateContentConfig(
|
85 |
+
system_instruction=bounding_box_system_instructions,
|
86 |
+
temperature=0.4,
|
87 |
+
safety_settings=safety_settings
|
88 |
+
)
|
89 |
+
)
|
90 |
+
|
91 |
+
json_data = extract_json_from_markdown(response.text)
|
92 |
+
boxes_with_labels, data = parse_info(resized_image, json_data)
|
93 |
+
return [resized_image, boxes_with_labels], data
|
94 |
+
|
95 |
+
iface = gr.Interface(
|
96 |
+
fn=estimate_co2,
|
97 |
+
inputs=gr.Image(type="pil"),
|
98 |
+
outputs=[
|
99 |
+
gr.AnnotatedImage(),
|
100 |
+
gr.Dataframe(
|
101 |
+
label="CO2 Estimation Data",
|
102 |
+
interactive=False,
|
103 |
+
headers=["co2", "item_name", "rationale"]
|
104 |
+
)
|
105 |
+
],
|
106 |
+
title="CO2 Estimation from Images",
|
107 |
+
description="Upload an image and get an estimation of the CO2 involved in the activities depicted.",
|
108 |
+
article="This is a very rough estimate, and can be misleading or factually inaccurate. Take this as a demo project and not as scientific/exact results."
|
109 |
+
#examples=[
|
110 |
+
# ["example.jpeg"] # Add an example image if you have one
|
111 |
+
#],
|
112 |
+
)
|
113 |
+
|
114 |
+
iface.launch()
|