osanseviero commited on
Commit
c2ba438
·
verified ·
1 Parent(s): 1774721

Create app.py

Browse files
Files changed (1) hide show
  1. app.py +114 -0
app.py ADDED
@@ -0,0 +1,114 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+ import gradio as gr
3
+ import pandas as pd
4
+ from google import genai
5
+
6
+ # Client and prompt setup
7
+ client = genai.Client(api_key=os.getenv('GOOGLE_API_KEY'))
8
+ model_name = "gemini-2.0-flash-exp" # Change to other models, but be careful as response might be with different structure
9
+ safety_settings = [
10
+ genai.types.SafetySetting(
11
+ category="HARM_CATEGORY_DANGEROUS_CONTENT",
12
+ threshold="BLOCK_ONLY_HIGH",
13
+ ),
14
+ ]
15
+ bounding_box_system_instructions = """Return bounding boxes as a JSON array with labels, CO2 estimate, and an explanation. Never return masks or code fencing. Limit to 5 objects."""
16
+ prompt = """Provide an estimation of how much CO2 is involved in all activities in this picture. Give CO2 in grams.
17
+
18
+ As examples, think of transport, smoking, meat, and other similar emission activities.
19
+ Do not provide actions that don't have CO2 emissions.
20
+
21
+ Be comprehensive, but don't list more than 10 objects. Detect the 2D bounding boxes of these activities,
22
+ including the label, the CO2 gram quantity, and a short explanation explaining the estimation
23
+ for each activity.
24
+ """
25
+
26
+ def parse_json(json_output):
27
+ # Based on https://github.com/google-gemini/cookbook/blob/main/gemini-2/spatial_understanding.ipynb
28
+ lines = json_output.splitlines()
29
+ for i, line in enumerate(lines):
30
+ if line == "```json":
31
+ json_output = "\n".join(lines[i+1:]) # Remove everything before "```json"
32
+ json_output = json_output.split("```")[0] # Remove everything after the closing "```"
33
+ break # Exit the loop once "```json" is found
34
+ return json_output
35
+
36
+ def parse_info(image, json_data):
37
+ width, height = image.size
38
+ df_data = []
39
+ boxes_with_labels = []
40
+
41
+ if not isinstance(json_data, list):
42
+ logging.error("JSON data is not a list as expected.")
43
+ return boxes_with_labels, pd.DataFrame(df_data)
44
+
45
+ # Iterate over each detected action actions
46
+ for action in info:
47
+ box_2d = action.get("box_2d")
48
+ label = action.get("label")
49
+ co2_grams = action.get("co2_grams")
50
+ explanation = action.get("explanation")
51
+
52
+ if not all([box_2d, label, co2_grams, explanation]):
53
+ continue
54
+
55
+ # Convert normalized coordinates to absolute coordinates
56
+ abs_y1 = int(box_2d[0] / 1000 * height)
57
+ abs_x1 = int(box_2d[1] / 1000 * width)
58
+ abs_y2 = int(box_2d[2] / 1000 * height)
59
+ abs_x2 = int(box_2d[3] / 1000 * width)
60
+
61
+ abs_x1, abs_x2 = min(abs_x1, abs_x2), max(abs_x1, abs_x2)
62
+ abs_y1, abs_y2 = min(abs_y1, abs_y2), max(abs_y1, abs_y2)
63
+
64
+ boxes_with_labels.append([(abs_x1, abs_y1, abs_x2, abs_y2), label])
65
+
66
+ df_data.append({
67
+ "label": label,
68
+ "co2": co2_grams,
69
+ "explanation": explanation
70
+ })
71
+
72
+ return boxes_with_labels, pd.DataFrame(df_data)
73
+
74
+ def estimate_co2(image):
75
+ resized_image = image.resize(
76
+ (1024, int(1024 * img.size[1] / img.size[0])),
77
+ Image.Resampling.LANCZOS
78
+ )
79
+
80
+ # Get resuls from model
81
+ response = client.models.generate_content(
82
+ model=model_name,
83
+ contents=[prompt, resized_image],
84
+ config = genai.types.GenerateContentConfig(
85
+ system_instruction=bounding_box_system_instructions,
86
+ temperature=0.4,
87
+ safety_settings=safety_settings
88
+ )
89
+ )
90
+
91
+ json_data = extract_json_from_markdown(response.text)
92
+ boxes_with_labels, data = parse_info(resized_image, json_data)
93
+ return [resized_image, boxes_with_labels], data
94
+
95
+ iface = gr.Interface(
96
+ fn=estimate_co2,
97
+ inputs=gr.Image(type="pil"),
98
+ outputs=[
99
+ gr.AnnotatedImage(),
100
+ gr.Dataframe(
101
+ label="CO2 Estimation Data",
102
+ interactive=False,
103
+ headers=["co2", "item_name", "rationale"]
104
+ )
105
+ ],
106
+ title="CO2 Estimation from Images",
107
+ description="Upload an image and get an estimation of the CO2 involved in the activities depicted.",
108
+ article="This is a very rough estimate, and can be misleading or factually inaccurate. Take this as a demo project and not as scientific/exact results."
109
+ #examples=[
110
+ # ["example.jpeg"] # Add an example image if you have one
111
+ #],
112
+ )
113
+
114
+ iface.launch()