Spaces:
Runtime error
Runtime error
Code refactoring
Browse files
app.py
CHANGED
@@ -18,16 +18,16 @@ from reportlab.lib.styles import getSampleStyleSheet
|
|
18 |
from reportlab.platypus import Image, SimpleDocTemplate, Table, TableStyle
|
19 |
from ultralytics import YOLO
|
20 |
|
21 |
-
with open(
|
22 |
config = json.load(f)
|
23 |
|
24 |
-
FACE_DET_TRESH = config[
|
25 |
-
FACE_DIST_TRESH = config[
|
26 |
-
YOLO_WEIGHTS_URL = config[
|
27 |
-
AVATARS_URI = config[
|
28 |
-
APP_NAME = config[
|
29 |
-
APP_DESCRIPTION = config[
|
30 |
-
APP_LOGO = config[
|
31 |
|
32 |
def load_detector():
|
33 |
yolo_weights_filename = os.path.basename(YOLO_WEIGHTS_URL)
|
@@ -43,18 +43,18 @@ model = load_detector()
|
|
43 |
|
44 |
styles = getSampleStyleSheet()
|
45 |
style_table = TableStyle([
|
46 |
-
(
|
47 |
-
(
|
48 |
-
(
|
49 |
-
(
|
50 |
-
(
|
51 |
-
(
|
52 |
-
(
|
53 |
-
(
|
54 |
])
|
55 |
|
56 |
def parse_album(data):
|
57 |
-
album_info = data[
|
58 |
album_info = album_info.split("-")[-1].split("_")
|
59 |
owner_id = - int(album_info[0])
|
60 |
album_id = int(album_info[1])
|
@@ -62,27 +62,27 @@ def parse_album(data):
|
|
62 |
|
63 |
def get_photos(owner_id, album_id, vk):
|
64 |
offset = 0
|
65 |
-
total_count = float(
|
66 |
count_per_request = 50
|
67 |
|
68 |
output = []
|
69 |
|
70 |
while offset < total_count:
|
71 |
params = {
|
72 |
-
|
73 |
-
|
74 |
-
|
75 |
-
|
76 |
-
|
77 |
}
|
78 |
|
79 |
response = vk.photos.get(**params)
|
80 |
|
81 |
-
for item in response[
|
82 |
-
max_item = max(item[
|
83 |
-
output.append(max_item[
|
84 |
|
85 |
-
total_count = response[
|
86 |
offset += count_per_request
|
87 |
|
88 |
return output
|
@@ -92,41 +92,41 @@ def download_images(photos, players):
|
|
92 |
folder_name = current_datetime.strftime("%Y-%m-%d_%H-%M-%S")
|
93 |
os.mkdir(folder_name)
|
94 |
|
95 |
-
players_path = os.path.join(folder_name,
|
96 |
-
photos_path = os.path.join(folder_name,
|
97 |
-
temp_path = os.path.join(folder_name,
|
98 |
|
99 |
os.mkdir(players_path)
|
100 |
os.mkdir(photos_path)
|
101 |
os.mkdir(temp_path)
|
102 |
|
103 |
-
update_progress(0,
|
104 |
for i, photo_url in enumerate(photos):
|
105 |
-
filename = f
|
106 |
response = requests.get(photo_url)
|
107 |
with open(os.path.join(photos_path, filename), "wb") as file:
|
108 |
file.write(response.content)
|
109 |
-
update_progress((i+1)/len(photos),
|
110 |
|
111 |
for team_state in players.keys():
|
112 |
update_progress(0, f"Downloading {team_state} players' avatars...")
|
113 |
for i, player in enumerate(players[team_state]):
|
114 |
filename = f"{player['id']}.jpg"
|
115 |
-
response = requests.get(player[
|
116 |
with open(os.path.join(players_path, filename), "wb") as file:
|
117 |
file.write(response.content)
|
118 |
update_progress((i+1)/len(players[team_state]), f"Downloading {team_state} players' avatars...")
|
119 |
|
120 |
return {
|
121 |
-
|
122 |
-
|
123 |
-
|
124 |
-
|
125 |
}
|
126 |
|
127 |
def find_photos(data, vk):
|
128 |
-
pattern = re.compile(
|
129 |
-
script_content = pattern.search(data).group(1).replace(
|
130 |
|
131 |
data = json.loads(script_content)
|
132 |
|
@@ -136,52 +136,52 @@ def find_photos(data, vk):
|
|
136 |
|
137 |
def translate(text):
|
138 |
translator = Translator()
|
139 |
-
output = translator.translate(text, src=
|
140 |
return output.text
|
141 |
|
142 |
def get_players(data):
|
143 |
output = {}
|
144 |
-
team_states = [
|
145 |
-
soup = BeautifulSoup(data,
|
146 |
|
147 |
for team_state in team_states:
|
148 |
-
update_progress(0, f
|
149 |
output[team_state] = []
|
150 |
player_roots = soup.find_all("div", {"class": f"{team_state} ng-star-inserted"})
|
151 |
for i, player_root in enumerate(player_roots):
|
152 |
player_info = player_root.find("a", {"class": "wrapper ng-star-inserted"})
|
153 |
-
id = re.findall(r
|
154 |
avatar_url = AVATARS_URI.replace("PLAYER_ID", id)
|
155 |
name = player_info.find("span", {"class": "name"}).get_text()
|
156 |
name = translate(name)
|
157 |
position = player_info.find("span", {"class": "position"}).get_text()
|
158 |
output[team_state].append({
|
159 |
-
|
160 |
-
|
161 |
-
|
162 |
-
|
163 |
})
|
164 |
-
update_progress((i+1)/len(player_roots), f
|
165 |
|
166 |
return output
|
167 |
|
168 |
def load_players_avatars(players, images_path, face_det_tresh):
|
169 |
for team_state in players.keys():
|
170 |
-
update_progress(0, f
|
171 |
for i, player in enumerate(players[team_state]):
|
172 |
image_name = f"{player['id']}.jpg"
|
173 |
-
player[
|
174 |
-
faces = find_faces(player[
|
175 |
if faces:
|
176 |
-
player[
|
177 |
|
178 |
-
update_progress((i+1)/len(players[team_state]), f
|
179 |
|
180 |
return players
|
181 |
|
182 |
def find_distance(base_face, check_face):
|
183 |
result = DeepFace.verify(base_face, check_face, enforce_detection=False)
|
184 |
-
return result[
|
185 |
|
186 |
def read_image_from_path(path):
|
187 |
return cv2.imread(path)
|
@@ -190,7 +190,7 @@ def read_images_from_path(path):
|
|
190 |
images = []
|
191 |
|
192 |
files = os.listdir(path)
|
193 |
-
update_progress(0,
|
194 |
|
195 |
for i, filename in enumerate(files):
|
196 |
if filename.endswith(".jpg"):
|
@@ -199,7 +199,7 @@ def read_images_from_path(path):
|
|
199 |
if image is not None:
|
200 |
images.append(image)
|
201 |
|
202 |
-
update_progress((i+1)/len(files),
|
203 |
|
204 |
return images
|
205 |
|
@@ -225,10 +225,10 @@ def find_faces(image, face_det_tresh):
|
|
225 |
def is_face_exists(players, face, face_dist_tresh):
|
226 |
for team_state in players.keys():
|
227 |
for player in players[team_state]:
|
228 |
-
if
|
229 |
-
distance = find_distance(player[
|
230 |
if distance <= face_dist_tresh:
|
231 |
-
return player[
|
232 |
return None, None
|
233 |
|
234 |
def add_players_table(elements, players):
|
@@ -238,12 +238,12 @@ def add_players_table(elements, players):
|
|
238 |
for team_state in players.keys():
|
239 |
update_progress(0, f"Creating dump of {team_state}'s squad...")
|
240 |
for i, player in enumerate(players[team_state]):
|
241 |
-
face = cv2_to_reportlab(player[
|
242 |
-
avatar = cv2_to_reportlab(player[
|
243 |
line = [
|
244 |
-
player[
|
245 |
-
player[
|
246 |
-
player[
|
247 |
avatar,
|
248 |
face
|
249 |
]
|
@@ -261,7 +261,7 @@ def check_faces(elements, photos, players, face_det_tresh, face_dist_tresh):
|
|
261 |
["Face", "Player ID", "Player Face"]
|
262 |
]
|
263 |
|
264 |
-
update_progress(0,
|
265 |
for i, photo in enumerate(photos):
|
266 |
faces = find_faces(photo, face_det_tresh)
|
267 |
for j, face in enumerate(faces):
|
@@ -272,7 +272,7 @@ def check_faces(elements, photos, players, face_det_tresh, face_dist_tresh):
|
|
272 |
player_face = cv2_to_reportlab(player_face)
|
273 |
tmp_arr.append(player_face)
|
274 |
data.append(tmp_arr)
|
275 |
-
update_progress((j+1)/len(faces), f
|
276 |
|
277 |
table = Table(data)
|
278 |
table.setStyle(style_table)
|
@@ -285,27 +285,26 @@ def update_progress(percent, description):
|
|
285 |
progress_bar.progress(percent)
|
286 |
progress_status_text.text(description)
|
287 |
|
288 |
-
|
289 |
def process(token, afl_link, face_dist_tresh, face_det_tresh):
|
290 |
-
update_progress(0,
|
291 |
vk_session = vk_api.VkApi(token=token)
|
292 |
vk = vk_session.get_api()
|
293 |
-
update_progress(100,
|
294 |
|
295 |
-
update_progress(0,
|
296 |
response = requests.get(afl_link)
|
297 |
-
update_progress(100,
|
298 |
|
299 |
-
update_progress(0,
|
300 |
photos = find_photos(response.text, vk)
|
301 |
-
update_progress(100,
|
302 |
|
303 |
players = get_players(response.text)
|
304 |
result = download_images(photos, players)
|
305 |
-
photos = read_images_from_path(result[
|
306 |
-
players = load_players_avatars(players, result[
|
307 |
|
308 |
-
table_file = os.path.join(result[
|
309 |
|
310 |
doc = SimpleDocTemplate(table_file, pagesize=letter)
|
311 |
|
@@ -315,10 +314,10 @@ def process(token, afl_link, face_dist_tresh, face_det_tresh):
|
|
315 |
|
316 |
doc.build(elements)
|
317 |
|
318 |
-
with open(table_file,
|
319 |
pdf_bytes = file.read()
|
320 |
|
321 |
-
shutil.rmtree(result[
|
322 |
|
323 |
return pdf_bytes
|
324 |
|
@@ -329,11 +328,11 @@ st.image(APP_LOGO, use_column_width=True)
|
|
329 |
|
330 |
st.write(APP_DESCRIPTION)
|
331 |
|
332 |
-
access_token = st.text_input("Your VK API access token", help=
|
333 |
-
afl_url = st.text_input("AFL url", help=
|
334 |
|
335 |
-
face_det_tresh = st.slider(
|
336 |
-
face_dist_tresh = st.slider(
|
337 |
|
338 |
button_clicked = st.button("Process")
|
339 |
|
@@ -341,4 +340,4 @@ if button_clicked:
|
|
341 |
progress_bar = st.progress(0)
|
342 |
progress_status_text = st.empty()
|
343 |
pdf_bytes = process(access_token, afl_url, face_dist_tresh, face_det_tresh)
|
344 |
-
st.download_button(label=
|
|
|
18 |
from reportlab.platypus import Image, SimpleDocTemplate, Table, TableStyle
|
19 |
from ultralytics import YOLO
|
20 |
|
21 |
+
with open("config.json", "r") as f:
|
22 |
config = json.load(f)
|
23 |
|
24 |
+
FACE_DET_TRESH = config["FACE_DET_TRESH"]
|
25 |
+
FACE_DIST_TRESH = config["FACE_DIST_TRESH"]
|
26 |
+
YOLO_WEIGHTS_URL = config["YOLO_WEIGHTS_URL"]
|
27 |
+
AVATARS_URI = config["AVATARS_URI"]
|
28 |
+
APP_NAME = config["APP_NAME"]
|
29 |
+
APP_DESCRIPTION = config["APP_DESCRIPTION"]
|
30 |
+
APP_LOGO = config["APP_LOGO"]
|
31 |
|
32 |
def load_detector():
|
33 |
yolo_weights_filename = os.path.basename(YOLO_WEIGHTS_URL)
|
|
|
43 |
|
44 |
styles = getSampleStyleSheet()
|
45 |
style_table = TableStyle([
|
46 |
+
("BACKGROUND", (0, 0), (-1, 0), colors.grey),
|
47 |
+
("TEXTCOLOR", (0, 0), (-1, 0), colors.whitesmoke),
|
48 |
+
("ALIGN", (0, 0), (-1, -1), "CENTER"),
|
49 |
+
("FONTNAME", (0, 0), (-1, 0), "Helvetica-Bold"),
|
50 |
+
("FONTSIZE", (0, 0), (-1, 0), 14),
|
51 |
+
("BOTTOMPADDING", (0, 0), (-1, 0), 12),
|
52 |
+
("BACKGROUND", (0, 1), (-1, -1), colors.beige),
|
53 |
+
("GRID", (0, 0), (-1, -1), 1, colors.black),
|
54 |
])
|
55 |
|
56 |
def parse_album(data):
|
57 |
+
album_info = data["NGRX_STATE"]["game"]["info"]["data"]["photoSetUrl"]
|
58 |
album_info = album_info.split("-")[-1].split("_")
|
59 |
owner_id = - int(album_info[0])
|
60 |
album_id = int(album_info[1])
|
|
|
62 |
|
63 |
def get_photos(owner_id, album_id, vk):
|
64 |
offset = 0
|
65 |
+
total_count = float("inf")
|
66 |
count_per_request = 50
|
67 |
|
68 |
output = []
|
69 |
|
70 |
while offset < total_count:
|
71 |
params = {
|
72 |
+
"owner_id": owner_id,
|
73 |
+
"album_id": album_id,
|
74 |
+
"count": count_per_request,
|
75 |
+
"offset": offset,
|
76 |
+
"extended": "1"
|
77 |
}
|
78 |
|
79 |
response = vk.photos.get(**params)
|
80 |
|
81 |
+
for item in response["items"]:
|
82 |
+
max_item = max(item["sizes"], key=lambda item: item["height"])
|
83 |
+
output.append(max_item["url"])
|
84 |
|
85 |
+
total_count = response["count"]
|
86 |
offset += count_per_request
|
87 |
|
88 |
return output
|
|
|
92 |
folder_name = current_datetime.strftime("%Y-%m-%d_%H-%M-%S")
|
93 |
os.mkdir(folder_name)
|
94 |
|
95 |
+
players_path = os.path.join(folder_name, "players")
|
96 |
+
photos_path = os.path.join(folder_name, "photos")
|
97 |
+
temp_path = os.path.join(folder_name, "temp")
|
98 |
|
99 |
os.mkdir(players_path)
|
100 |
os.mkdir(photos_path)
|
101 |
os.mkdir(temp_path)
|
102 |
|
103 |
+
update_progress(0, "Downloading photos...")
|
104 |
for i, photo_url in enumerate(photos):
|
105 |
+
filename = f"{i}.jpg"
|
106 |
response = requests.get(photo_url)
|
107 |
with open(os.path.join(photos_path, filename), "wb") as file:
|
108 |
file.write(response.content)
|
109 |
+
update_progress((i+1)/len(photos), "Downloading photos...")
|
110 |
|
111 |
for team_state in players.keys():
|
112 |
update_progress(0, f"Downloading {team_state} players' avatars...")
|
113 |
for i, player in enumerate(players[team_state]):
|
114 |
filename = f"{player['id']}.jpg"
|
115 |
+
response = requests.get(player["avatar_url"])
|
116 |
with open(os.path.join(players_path, filename), "wb") as file:
|
117 |
file.write(response.content)
|
118 |
update_progress((i+1)/len(players[team_state]), f"Downloading {team_state} players' avatars...")
|
119 |
|
120 |
return {
|
121 |
+
"photos_path": photos_path,
|
122 |
+
"players_path": players_path,
|
123 |
+
"temp_path": temp_path,
|
124 |
+
"folder_name": folder_name
|
125 |
}
|
126 |
|
127 |
def find_photos(data, vk):
|
128 |
+
pattern = re.compile("<script id='axl-desktop-state' type='application/json'>(.+?)</script>")
|
129 |
+
script_content = pattern.search(data).group(1).replace("&q;", "")
|
130 |
|
131 |
data = json.loads(script_content)
|
132 |
|
|
|
136 |
|
137 |
def translate(text):
|
138 |
translator = Translator()
|
139 |
+
output = translator.translate(text, src="ru", dest="en")
|
140 |
return output.text
|
141 |
|
142 |
def get_players(data):
|
143 |
output = {}
|
144 |
+
team_states = ["home", "away"]
|
145 |
+
soup = BeautifulSoup(data, "lxml")
|
146 |
|
147 |
for team_state in team_states:
|
148 |
+
update_progress(0, f"Getting information about {team_state} players...")
|
149 |
output[team_state] = []
|
150 |
player_roots = soup.find_all("div", {"class": f"{team_state} ng-star-inserted"})
|
151 |
for i, player_root in enumerate(player_roots):
|
152 |
player_info = player_root.find("a", {"class": "wrapper ng-star-inserted"})
|
153 |
+
id = re.findall(r"\d+", player_info["href"])[-1]
|
154 |
avatar_url = AVATARS_URI.replace("PLAYER_ID", id)
|
155 |
name = player_info.find("span", {"class": "name"}).get_text()
|
156 |
name = translate(name)
|
157 |
position = player_info.find("span", {"class": "position"}).get_text()
|
158 |
output[team_state].append({
|
159 |
+
"id": id,
|
160 |
+
"name": name,
|
161 |
+
"position": position,
|
162 |
+
"avatar_url": avatar_url
|
163 |
})
|
164 |
+
update_progress((i+1)/len(player_roots), f"Getting information about {team_state} players...")
|
165 |
|
166 |
return output
|
167 |
|
168 |
def load_players_avatars(players, images_path, face_det_tresh):
|
169 |
for team_state in players.keys():
|
170 |
+
update_progress(0, f"Reading avatars of {team_state} team...")
|
171 |
for i, player in enumerate(players[team_state]):
|
172 |
image_name = f"{player['id']}.jpg"
|
173 |
+
player["image"] = read_image_from_path(os.path.join(images_path, image_name))
|
174 |
+
faces = find_faces(player["image"], face_det_tresh)
|
175 |
if faces:
|
176 |
+
player["face"] = faces[0]
|
177 |
|
178 |
+
update_progress((i+1)/len(players[team_state]), f"Reading avatars of {team_state} team...")
|
179 |
|
180 |
return players
|
181 |
|
182 |
def find_distance(base_face, check_face):
|
183 |
result = DeepFace.verify(base_face, check_face, enforce_detection=False)
|
184 |
+
return result["distance"]
|
185 |
|
186 |
def read_image_from_path(path):
|
187 |
return cv2.imread(path)
|
|
|
190 |
images = []
|
191 |
|
192 |
files = os.listdir(path)
|
193 |
+
update_progress(0, "Reading photos...")
|
194 |
|
195 |
for i, filename in enumerate(files):
|
196 |
if filename.endswith(".jpg"):
|
|
|
199 |
if image is not None:
|
200 |
images.append(image)
|
201 |
|
202 |
+
update_progress((i+1)/len(files), "Reading photos...")
|
203 |
|
204 |
return images
|
205 |
|
|
|
225 |
def is_face_exists(players, face, face_dist_tresh):
|
226 |
for team_state in players.keys():
|
227 |
for player in players[team_state]:
|
228 |
+
if "face" in player:
|
229 |
+
distance = find_distance(player["face"], face)
|
230 |
if distance <= face_dist_tresh:
|
231 |
+
return player["id"], player["face"]
|
232 |
return None, None
|
233 |
|
234 |
def add_players_table(elements, players):
|
|
|
238 |
for team_state in players.keys():
|
239 |
update_progress(0, f"Creating dump of {team_state}'s squad...")
|
240 |
for i, player in enumerate(players[team_state]):
|
241 |
+
face = cv2_to_reportlab(player["face"]) if "face" in player else None
|
242 |
+
avatar = cv2_to_reportlab(player["image"])
|
243 |
line = [
|
244 |
+
player["id"],
|
245 |
+
player["name"],
|
246 |
+
player["position"],
|
247 |
avatar,
|
248 |
face
|
249 |
]
|
|
|
261 |
["Face", "Player ID", "Player Face"]
|
262 |
]
|
263 |
|
264 |
+
update_progress(0, "Comparing faces...")
|
265 |
for i, photo in enumerate(photos):
|
266 |
faces = find_faces(photo, face_det_tresh)
|
267 |
for j, face in enumerate(faces):
|
|
|
272 |
player_face = cv2_to_reportlab(player_face)
|
273 |
tmp_arr.append(player_face)
|
274 |
data.append(tmp_arr)
|
275 |
+
update_progress((j+1)/len(faces), f"[{i + 1}/{len(photos)}] Comparing faces...")
|
276 |
|
277 |
table = Table(data)
|
278 |
table.setStyle(style_table)
|
|
|
285 |
progress_bar.progress(percent)
|
286 |
progress_status_text.text(description)
|
287 |
|
|
|
288 |
def process(token, afl_link, face_dist_tresh, face_det_tresh):
|
289 |
+
update_progress(0, "Connecting to vk...")
|
290 |
vk_session = vk_api.VkApi(token=token)
|
291 |
vk = vk_session.get_api()
|
292 |
+
update_progress(100, "Connected to vk")
|
293 |
|
294 |
+
update_progress(0, "Getting information from afl...")
|
295 |
response = requests.get(afl_link)
|
296 |
+
update_progress(100, "Got information from afl")
|
297 |
|
298 |
+
update_progress(0, "Getting information about photos...")
|
299 |
photos = find_photos(response.text, vk)
|
300 |
+
update_progress(100, "Got information about photos")
|
301 |
|
302 |
players = get_players(response.text)
|
303 |
result = download_images(photos, players)
|
304 |
+
photos = read_images_from_path(result["photos_path"])
|
305 |
+
players = load_players_avatars(players, result["players_path"], face_det_tresh)
|
306 |
|
307 |
+
table_file = os.path.join(result["temp_path"], "table.pdf")
|
308 |
|
309 |
doc = SimpleDocTemplate(table_file, pagesize=letter)
|
310 |
|
|
|
314 |
|
315 |
doc.build(elements)
|
316 |
|
317 |
+
with open(table_file, "rb") as file:
|
318 |
pdf_bytes = file.read()
|
319 |
|
320 |
+
shutil.rmtree(result["folder_name"])
|
321 |
|
322 |
return pdf_bytes
|
323 |
|
|
|
328 |
|
329 |
st.write(APP_DESCRIPTION)
|
330 |
|
331 |
+
access_token = st.text_input("Your VK API access token", help="You can obtain your token from https://vkhost.github.io/")
|
332 |
+
afl_url = st.text_input("AFL url", help="Example: https://afl.ru/football/afl-moscow-8x8/afl-cup-krasnaya-presnya-3097/matches/463676")
|
333 |
|
334 |
+
face_det_tresh = st.slider("face_det_tresh:", 0.0, 1.0, FACE_DET_TRESH, 0.01, help="Adjust the threshold value for face detection.")
|
335 |
+
face_dist_tresh = st.slider("face_dist_tresh:", 0.0, 1.0, FACE_DIST_TRESH, 0.01, help="Adjust the threshold to determine the minimum acceptable distance between faces.")
|
336 |
|
337 |
button_clicked = st.button("Process")
|
338 |
|
|
|
340 |
progress_bar = st.progress(0)
|
341 |
progress_status_text = st.empty()
|
342 |
pdf_bytes = process(access_token, afl_url, face_dist_tresh, face_det_tresh)
|
343 |
+
st.download_button(label="Download PDF", data=pdf_bytes, file_name="output.pdf")
|